Package gluon :: Module dal
[hide private]
[frames] | no frames]

Source Code for Module gluon.dal

    1  #!/bin/env python 
    2  # -*- coding: utf-8 -*- 
    3   
    4  """ 
    5  This file is part of the web2py Web Framework 
    6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
    7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 
    8   
    9  Thanks to 
   10      * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support 
   11      * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support 
   12      * Denes 
   13      * Chris Clark 
   14      * clach05 
   15      * Denes Lengyel 
   16      * and many others who have contributed to current and previous versions 
   17   
   18  This file contains the DAL support for many relational databases, 
   19  including: 
   20  - SQLite & SpatiaLite 
   21  - MySQL 
   22  - Postgres 
   23  - Firebird 
   24  - Oracle 
   25  - MS SQL 
   26  - DB2 
   27  - Interbase 
   28  - Ingres 
   29  - Informix (9+ and SE) 
   30  - SapDB (experimental) 
   31  - Cubrid (experimental) 
   32  - CouchDB (experimental) 
   33  - MongoDB (in progress) 
   34  - Google:nosql 
   35  - Google:sql 
   36  - Teradata 
   37  - IMAP (experimental) 
   38   
   39  Example of usage: 
   40   
   41  >>> # from dal import DAL, Field 
   42   
   43  ### create DAL connection (and create DB if it doesn't exist) 
   44  >>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'), 
   45  ... folder=None) 
   46   
   47  ### define a table 'person' (create/alter as necessary) 
   48  >>> person = db.define_table('person',Field('name','string')) 
   49   
   50  ### insert a record 
   51  >>> id = person.insert(name='James') 
   52   
   53  ### retrieve it by id 
   54  >>> james = person(id) 
   55   
   56  ### retrieve it by name 
   57  >>> james = person(name='James') 
   58   
   59  ### retrieve it by arbitrary query 
   60  >>> query = (person.name=='James') & (person.name.startswith('J')) 
   61  >>> james = db(query).select(person.ALL)[0] 
   62   
   63  ### update one record 
   64  >>> james.update_record(name='Jim') 
   65  <Row {'id': 1, 'name': 'Jim'}> 
   66   
   67  ### update multiple records by query 
   68  >>> db(person.name.like('J%')).update(name='James') 
   69  1 
   70   
   71  ### delete records by query 
   72  >>> db(person.name.lower() == 'jim').delete() 
   73  0 
   74   
   75  ### retrieve multiple records (rows) 
   76  >>> people = db(person).select(orderby=person.name, 
   77  ... groupby=person.name, limitby=(0,100)) 
   78   
   79  ### further filter them 
   80  >>> james = people.find(lambda row: row.name == 'James').first() 
   81  >>> print james.id, james.name 
   82  1 James 
   83   
   84  ### check aggregates 
   85  >>> counter = person.id.count() 
   86  >>> print db(person).select(counter).first()(counter) 
   87  1 
   88   
   89  ### delete one record 
   90  >>> james.delete_record() 
   91  1 
   92   
   93  ### delete (drop) entire database table 
   94  >>> person.drop() 
   95   
   96  Supported field types: 
   97  id string text boolean integer double decimal password upload 
   98  blob time date datetime 
   99   
  100  Supported DAL URI strings: 
  101  'sqlite://test.db' 
  102  'spatialite://test.db' 
  103  'sqlite:memory' 
  104  'spatialite:memory' 
  105  'jdbc:sqlite://test.db' 
  106  'mysql://root:none@localhost/test' 
  107  'postgres://mdipierro:password@localhost/test' 
  108  'postgres:psycopg2://mdipierro:password@localhost/test' 
  109  'postgres:pg8000://mdipierro:password@localhost/test' 
  110  'jdbc:postgres://mdipierro:none@localhost/test' 
  111  'mssql://web2py:none@A64X2/web2py_test' 
  112  'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings 
  113  'oracle://username:password@database' 
  114  'firebird://user:password@server:3050/database' 
  115  'db2://DSN=dsn;UID=user;PWD=pass' 
  116  'firebird://username:password@hostname/database' 
  117  'firebird_embedded://username:password@c://path' 
  118  'informix://user:password@server:3050/database' 
  119  'informixu://user:password@server:3050/database' # unicode informix 
  120  'ingres://database'  # or use an ODBC connection string, e.g. 'ingres://dsn=dsn_name' 
  121  'google:datastore' # for google app engine datastore 
  122  'google:sql' # for google app engine with sql (mysql compatible) 
  123  'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental 
  124  'imap://user:password@server:port' # experimental 
  125  'mongodb://user:password@server:port/database' # experimental 
  126   
  127  For more info: 
  128  help(DAL) 
  129  help(Field) 
  130  """ 
  131   
  132  ################################################################################### 
  133  # this file only exposes DAL and Field 
  134  ################################################################################### 
  135   
  136  __all__ = ['DAL', 'Field'] 
  137   
  138  DEFAULTLENGTH = {'string':512, 
  139                   'password':512, 
  140                   'upload':512, 
  141                   'text':2**15, 
  142                   'blob':2**31} 
  143  TIMINGSSIZE = 100 
  144  SPATIALLIBS = { 
  145      'Windows':'libspatialite', 
  146      'Linux':'libspatialite.so', 
  147      'Darwin':'libspatialite.dylib' 
  148      } 
  149  DEFAULT_URI = 'sqlite://dummy.db' 
  150   
  151  import re 
  152  import sys 
  153  import locale 
  154  import os 
  155  import types 
  156  import datetime 
  157  import threading 
  158  import time 
  159  import csv 
  160  import cgi 
  161  import copy 
  162  import socket 
  163  import logging 
  164  import base64 
  165  import shutil 
  166  import marshal 
  167  import decimal 
  168  import struct 
  169  import urllib 
  170  import hashlib 
  171  import uuid 
  172  import glob 
  173  import traceback 
  174  import platform 
  175   
  176  PYTHON_VERSION = sys.version_info[0] 
  177  if PYTHON_VERSION == 2: 
  178      import cPickle as pickle 
  179      import cStringIO as StringIO 
  180      import copy_reg as copyreg 
  181      hashlib_md5 = hashlib.md5 
  182      bytes, unicode = str, unicode 
  183  else: 
  184      import pickle 
  185      from io import StringIO as StringIO 
  186      import copyreg 
  187      long = int 
  188      hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8')) 
  189      bytes, unicode = bytes, str 
  190   
  191  CALLABLETYPES = (types.LambdaType, types.FunctionType, 
  192                   types.BuiltinFunctionType, 
  193                   types.MethodType, types.BuiltinMethodType) 
  194   
  195  TABLE_ARGS = set( 
  196      ('migrate','primarykey','fake_migrate','format','redefine', 
  197       'singular','plural','trigger_name','sequence_name','fields', 
  198       'common_filter','polymodel','table_class','on_define','actual_name')) 
  199   
  200  SELECT_ARGS = set( 
  201      ('orderby', 'groupby', 'limitby','required', 'cache', 'left', 
  202       'distinct', 'having', 'join','for_update', 'processor','cacheable', 'orderby_on_limitby')) 
  203   
  204  ogetattr = object.__getattribute__ 
  205  osetattr = object.__setattr__ 
  206  exists = os.path.exists 
  207  pjoin = os.path.join 
  208   
  209  ################################################################################### 
  210  # following checks allow the use of dal without web2py, as a standalone module 
  211  ################################################################################### 
  212  try: 
  213      from gluon.utils import web2py_uuid 
  214  except (ImportError, SystemError): 
  215      import uuid 
216 - def web2py_uuid(): return str(uuid.uuid4())
217 218 try: 219 import portalocker 220 have_portalocker = True 221 except ImportError: 222 have_portalocker = False 223 224 try: 225 from gluon import serializers 226 have_serializers = True 227 except ImportError: 228 have_serializers = False 229 try: 230 import json as simplejson 231 except ImportError: 232 try: 233 import gluon.contrib.simplejson as simplejson 234 except ImportError: 235 simplejson = None 236 237 LOGGER = logging.getLogger("web2py.dal") 238 DEFAULT = lambda:0 239 240 GLOBAL_LOCKER = threading.RLock() 241 THREAD_LOCAL = threading.local() 242 243 # internal representation of tables with field 244 # <table>.<field>, tables and fields may only be [a-zA-Z0-9_] 245 246 REGEX_TYPE = re.compile('^([\w\_\:]+)') 247 REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*') 248 REGEX_W = re.compile('^\w+$') 249 REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$') 250 REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)(\.(?P<name>\w+))?\.\w+$') 251 REGEX_CLEANUP_FN = re.compile('[\'"\s;]+') 252 REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)') 253 REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$') 254 REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)") 255 REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')') 256 REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$') 257 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$') 258 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$') 259 REGEX_QUOTES = re.compile("'[^']*'") 260 REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$') 261 REGEX_PASSWORD = re.compile('\://([^:@]*)\:') 262 REGEX_NOPASSWD = re.compile('\/\/[\w\.\-]+[\:\/](.+)(?=@)') # was '(?<=[\:\/])([^:@/]+)(?=@.+)' 263 264 # list of drivers will be built on the fly 265 # and lists only what is available 266 DRIVERS = [] 267 268 try: 269 from new import classobj 270 from google.appengine.ext import db as gae 271 from google.appengine.api import namespace_manager, rdbms 272 from google.appengine.api.datastore_types import Key ### for belongs on ID 273 from google.appengine.ext.db.polymodel import PolyModel 274 DRIVERS.append('google') 275 except ImportError: 276 pass 277 278 if not 'google' in DRIVERS: 279 280 try: 281 from pysqlite2 import dbapi2 as sqlite2 282 DRIVERS.append('SQLite(sqlite2)') 283 except ImportError: 284 LOGGER.debug('no SQLite drivers pysqlite2.dbapi2') 285 286 try: 287 from sqlite3 import dbapi2 as sqlite3 288 DRIVERS.append('SQLite(sqlite3)') 289 except ImportError: 290 LOGGER.debug('no SQLite drivers sqlite3') 291 292 try: 293 # first try contrib driver, then from site-packages (if installed) 294 try: 295 import gluon.contrib.pymysql as pymysql 296 # monkeypatch pymysql because they havent fixed the bug: 297 # https://github.com/petehunt/PyMySQL/issues/86 298 pymysql.ESCAPE_REGEX = re.compile("'") 299 pymysql.ESCAPE_MAP = {"'": "''"} 300 # end monkeypatch 301 except ImportError: 302 import pymysql 303 DRIVERS.append('MySQL(pymysql)') 304 except ImportError: 305 LOGGER.debug('no MySQL driver pymysql') 306 307 try: 308 import MySQLdb 309 DRIVERS.append('MySQL(MySQLdb)') 310 except ImportError: 311 LOGGER.debug('no MySQL driver MySQLDB') 312 313 try: 314 import mysql.connector as mysqlconnector 315 DRIVERS.append("MySQL(mysqlconnector)") 316 except ImportError: 317 LOGGER.debug("no driver mysql.connector") 318 319 try: 320 import psycopg2 321 from psycopg2.extensions import adapt as psycopg2_adapt 322 DRIVERS.append('PostgreSQL(psycopg2)') 323 except ImportError: 324 LOGGER.debug('no PostgreSQL driver psycopg2') 325 326 try: 327 # first try contrib driver, then from site-packages (if installed) 328 try: 329 import gluon.contrib.pg8000.dbapi as pg8000 330 except ImportError: 331 import pg8000.dbapi as pg8000 332 DRIVERS.append('PostgreSQL(pg8000)') 333 except ImportError: 334 LOGGER.debug('no PostgreSQL driver pg8000') 335 336 try: 337 import cx_Oracle 338 DRIVERS.append('Oracle(cx_Oracle)') 339 except ImportError: 340 LOGGER.debug('no Oracle driver cx_Oracle') 341 342 try: 343 try: 344 import pyodbc 345 except ImportError: 346 try: 347 import gluon.contrib.pypyodbc as pyodbc 348 except Exception, e: 349 raise ImportError(str(e)) 350 DRIVERS.append('MSSQL(pyodbc)') 351 DRIVERS.append('DB2(pyodbc)') 352 DRIVERS.append('Teradata(pyodbc)') 353 DRIVERS.append('Ingres(pyodbc)') 354 except ImportError: 355 LOGGER.debug('no MSSQL/DB2/Teradata/Ingres driver pyodbc') 356 357 try: 358 import Sybase 359 DRIVERS.append('Sybase(Sybase)') 360 except ImportError: 361 LOGGER.debug('no Sybase driver') 362 363 try: 364 import kinterbasdb 365 DRIVERS.append('Interbase(kinterbasdb)') 366 DRIVERS.append('Firebird(kinterbasdb)') 367 except ImportError: 368 LOGGER.debug('no Firebird/Interbase driver kinterbasdb') 369 370 try: 371 import fdb 372 DRIVERS.append('Firebird(fdb)') 373 except ImportError: 374 LOGGER.debug('no Firebird driver fdb') 375 ##### 376 try: 377 import firebirdsql 378 DRIVERS.append('Firebird(firebirdsql)') 379 except ImportError: 380 LOGGER.debug('no Firebird driver firebirdsql') 381 382 try: 383 import informixdb 384 DRIVERS.append('Informix(informixdb)') 385 LOGGER.warning('Informix support is experimental') 386 except ImportError: 387 LOGGER.debug('no Informix driver informixdb') 388 389 try: 390 import sapdb 391 DRIVERS.append('SQL(sapdb)') 392 LOGGER.warning('SAPDB support is experimental') 393 except ImportError: 394 LOGGER.debug('no SAP driver sapdb') 395 396 try: 397 import cubriddb 398 DRIVERS.append('Cubrid(cubriddb)') 399 LOGGER.warning('Cubrid support is experimental') 400 except ImportError: 401 LOGGER.debug('no Cubrid driver cubriddb') 402 403 try: 404 from com.ziclix.python.sql import zxJDBC 405 import java.sql 406 # Try sqlite jdbc driver from http://www.zentus.com/sqlitejdbc/ 407 from org.sqlite import JDBC # required by java.sql; ensure we have it 408 zxJDBC_sqlite = java.sql.DriverManager 409 DRIVERS.append('PostgreSQL(zxJDBC)') 410 DRIVERS.append('SQLite(zxJDBC)') 411 LOGGER.warning('zxJDBC support is experimental') 412 is_jdbc = True 413 except ImportError: 414 LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC') 415 is_jdbc = False 416 417 try: 418 import couchdb 419 DRIVERS.append('CouchDB(couchdb)') 420 except ImportError: 421 LOGGER.debug('no Couchdb driver couchdb') 422 423 try: 424 import pymongo 425 DRIVERS.append('MongoDB(pymongo)') 426 except: 427 LOGGER.debug('no MongoDB driver pymongo') 428 429 try: 430 import imaplib 431 DRIVERS.append('IMAP(imaplib)') 432 except: 433 LOGGER.debug('no IMAP driver imaplib') 434 435 PLURALIZE_RULES = [ 436 (re.compile('child$'), re.compile('child$'), 'children'), 437 (re.compile('oot$'), re.compile('oot$'), 'eet'), 438 (re.compile('ooth$'), re.compile('ooth$'), 'eeth'), 439 (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'), 440 (re.compile('sis$'), re.compile('sis$'), 'ses'), 441 (re.compile('man$'), re.compile('man$'), 'men'), 442 (re.compile('ife$'), re.compile('ife$'), 'ives'), 443 (re.compile('eau$'), re.compile('eau$'), 'eaux'), 444 (re.compile('lf$'), re.compile('lf$'), 'lves'), 445 (re.compile('[sxz]$'), re.compile('$'), 'es'), 446 (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'), 447 (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'), 448 (re.compile('$'), re.compile('$'), 's'), 449 ]
450 451 -def pluralize(singular, rules=PLURALIZE_RULES):
452 for line in rules: 453 re_search, re_sub, replace = line 454 plural = re_search.search(singular) and re_sub.sub(replace, singular) 455 if plural: return plural
456
457 -def hide_password(uri):
458 if isinstance(uri,(list,tuple)): 459 return [hide_password(item) for item in uri] 460 return REGEX_NOPASSWD.sub('******',uri)
461
462 -def OR(a,b):
463 return a|b
464
465 -def AND(a,b):
466 return a&b
467
468 -def IDENTITY(x): return x
469
470 -def varquote_aux(name,quotestr='%s'):
471 return name if REGEX_W.match(name) else quotestr % name
472
473 -def quote_keyword(a,keyword='timestamp'):
474 regex = re.compile('\.keyword(?=\w)') 475 a = regex.sub('."%s"' % keyword,a) 476 return a
477 478 if 'google' in DRIVERS: 479 480 is_jdbc = False
481 482 - class GAEDecimalProperty(gae.Property):
483 """ 484 GAE decimal implementation 485 """ 486 data_type = decimal.Decimal 487
488 - def __init__(self, precision, scale, **kwargs):
489 super(GAEDecimalProperty, self).__init__(self, **kwargs) 490 d = '1.' 491 for x in range(scale): 492 d += '0' 493 self.round = decimal.Decimal(d)
494
495 - def get_value_for_datastore(self, model_instance):
496 value = super(GAEDecimalProperty, self)\ 497 .get_value_for_datastore(model_instance) 498 if value is None or value == '': 499 return None 500 else: 501 return str(value)
502
503 - def make_value_from_datastore(self, value):
504 if value is None or value == '': 505 return None 506 else: 507 return decimal.Decimal(value).quantize(self.round)
508
509 - def validate(self, value):
510 value = super(GAEDecimalProperty, self).validate(value) 511 if value is None or isinstance(value, decimal.Decimal): 512 return value 513 elif isinstance(value, basestring): 514 return decimal.Decimal(value) 515 raise gae.BadValueError("Property %s must be a Decimal or string."\ 516 % self.name)
517
518 ################################################################################### 519 # class that handles connection pooling (all adapters are derived from this one) 520 ################################################################################### 521 522 -class ConnectionPool(object):
523 524 POOLS = {} 525 check_active_connection = True 526 527 @staticmethod
528 - def set_folder(folder):
530 531 # ## this allows gluon to commit/rollback all dbs in this thread 532
533 - def close(self,action='commit',really=True):
534 if action: 535 if callable(action): 536 action(self) 537 else: 538 getattr(self, action)() 539 # ## if you want pools, recycle this connection 540 if self.pool_size: 541 GLOBAL_LOCKER.acquire() 542 pool = ConnectionPool.POOLS[self.uri] 543 if len(pool) < self.pool_size: 544 pool.append(self.connection) 545 really = False 546 GLOBAL_LOCKER.release() 547 if really: 548 self.close_connection() 549 self.connection = None
550 551 @staticmethod
552 - def close_all_instances(action):
553 """ to close cleanly databases in a multithreaded environment """ 554 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items() 555 for db_uid, db_group in dbs: 556 for db in db_group: 557 if hasattr(db,'_adapter'): 558 db._adapter.close(action) 559 getattr(THREAD_LOCAL,'db_instances',{}).clear() 560 getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear() 561 if callable(action): 562 action(None) 563 return
564
565 - def find_or_make_work_folder(self):
566 """ this actually does not make the folder. it has to be there """ 567 self.folder = getattr(THREAD_LOCAL,'folder','') 568 569 if (os.path.isabs(self.folder) and 570 isinstance(self, UseDatabaseStoredFile) and 571 self.folder.startswith(os.getcwd())): 572 self.folder = os.path.relpath(self.folder, os.getcwd()) 573 574 # Creating the folder if it does not exist 575 if False and self.folder and not exists(self.folder): 576 os.mkdir(self.folder)
577
578 - def after_connection_hook(self):
579 """hook for the after_connection parameter""" 580 if callable(self._after_connection): 581 self._after_connection(self) 582 self.after_connection()
583
584 - def after_connection(self):
585 """ this it is supposed to be overloaded by adapters""" 586 pass
587
588 - def reconnect(self, f=None, cursor=True):
589 """ 590 this function defines: self.connection and self.cursor 591 (iff cursor is True) 592 if self.pool_size>0 it will try pull the connection from the pool 593 if the connection is not active (closed by db server) it will loop 594 if not self.pool_size or no active connections in pool makes a new one 595 """ 596 if getattr(self,'connection', None) != None: 597 return 598 if f is None: 599 f = self.connector 600 601 # if not hasattr(self, "driver") or self.driver is None: 602 # LOGGER.debug("Skipping connection since there's no driver") 603 # return 604 605 if not self.pool_size: 606 self.connection = f() 607 self.cursor = cursor and self.connection.cursor() 608 else: 609 uri = self.uri 610 POOLS = ConnectionPool.POOLS 611 while True: 612 GLOBAL_LOCKER.acquire() 613 if not uri in POOLS: 614 POOLS[uri] = [] 615 if POOLS[uri]: 616 self.connection = POOLS[uri].pop() 617 GLOBAL_LOCKER.release() 618 self.cursor = cursor and self.connection.cursor() 619 try: 620 if self.cursor and self.check_active_connection: 621 self.execute('SELECT 1;') 622 break 623 except: 624 pass 625 else: 626 GLOBAL_LOCKER.release() 627 self.connection = f() 628 self.cursor = cursor and self.connection.cursor() 629 break 630 self.after_connection_hook()
631
632 633 ################################################################################### 634 # this is a generic adapter that does nothing; all others are derived from this one 635 ################################################################################### 636 637 -class BaseAdapter(ConnectionPool):
638 native_json = False 639 driver = None 640 driver_name = None 641 drivers = () # list of drivers from which to pick 642 connection = None 643 commit_on_alter_table = False 644 support_distributed_transaction = False 645 uploads_in_blob = False 646 can_select_for_update = True 647 dbpath = None 648 folder = None 649 650 TRUE = 'T' 651 FALSE = 'F' 652 T_SEP = ' ' 653 QUOTE_TEMPLATE = '"%s"' 654 655 types = { 656 'boolean': 'CHAR(1)', 657 'string': 'CHAR(%(length)s)', 658 'text': 'TEXT', 659 'json': 'TEXT', 660 'password': 'CHAR(%(length)s)', 661 'blob': 'BLOB', 662 'upload': 'CHAR(%(length)s)', 663 'integer': 'INTEGER', 664 'bigint': 'INTEGER', 665 'float':'DOUBLE', 666 'double': 'DOUBLE', 667 'decimal': 'DOUBLE', 668 'date': 'DATE', 669 'time': 'TIME', 670 'datetime': 'TIMESTAMP', 671 'id': 'INTEGER PRIMARY KEY AUTOINCREMENT', 672 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 673 'list:integer': 'TEXT', 674 'list:string': 'TEXT', 675 'list:reference': 'TEXT', 676 # the two below are only used when DAL(...bigint_id=True) and replace 'id','reference' 677 'big-id': 'BIGINT PRIMARY KEY AUTOINCREMENT', 678 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 679 } 680
681 - def isOperationalError(self,exception):
682 if not hasattr(self.driver, "OperationalError"): 683 return None 684 return isinstance(exception, self.driver.OperationalError)
685
686 - def isProgrammingError(self,exception):
687 if not hasattr(self.driver, "ProgrammingError"): 688 return None 689 return isinstance(exception, self.driver.ProgrammingError)
690
691 - def id_query(self, table):
692 pkeys = getattr(table,'_primarykey',None) 693 if pkeys: 694 return table[pkeys[0]] != None 695 else: 696 return table._id != None
697
698 - def adapt(self, obj):
699 return "'%s'" % obj.replace("'", "''")
700
701 - def smart_adapt(self, obj):
702 if isinstance(obj,(int,float)): 703 return str(obj) 704 return self.adapt(str(obj))
705
706 - def file_exists(self, filename):
707 """ 708 to be used ONLY for files that on GAE may not be on filesystem 709 """ 710 return exists(filename)
711
712 - def file_open(self, filename, mode='rb', lock=True):
713 """ 714 to be used ONLY for files that on GAE may not be on filesystem 715 """ 716 if have_portalocker and lock: 717 fileobj = portalocker.LockedFile(filename,mode) 718 else: 719 fileobj = open(filename,mode) 720 return fileobj
721
722 - def file_close(self, fileobj):
723 """ 724 to be used ONLY for files that on GAE may not be on filesystem 725 """ 726 if fileobj: 727 fileobj.close()
728
729 - def file_delete(self, filename):
730 os.unlink(filename)
731
732 - def find_driver(self,adapter_args,uri=None):
733 self.adapter_args = adapter_args 734 if getattr(self,'driver',None) != None: 735 return 736 drivers_available = [driver for driver in self.drivers 737 if driver in globals()] 738 if uri: 739 items = uri.split('://',1)[0].split(':') 740 request_driver = items[1] if len(items)>1 else None 741 else: 742 request_driver = None 743 request_driver = request_driver or adapter_args.get('driver') 744 if request_driver: 745 if request_driver in drivers_available: 746 self.driver_name = request_driver 747 self.driver = globals().get(request_driver) 748 else: 749 raise RuntimeError("driver %s not available" % request_driver) 750 elif drivers_available: 751 self.driver_name = drivers_available[0] 752 self.driver = globals().get(self.driver_name) 753 else: 754 raise RuntimeError("no driver available %s" % str(self.drivers))
755
756 - def log(self, message, table=None):
757 """ Logs migrations 758 759 It will not log changes if logfile is not specified. Defaults 760 to sql.log 761 """ 762 763 isabs = None 764 logfilename = self.adapter_args.get('logfile','sql.log') 765 writelog = bool(logfilename) 766 if writelog: 767 isabs = os.path.isabs(logfilename) 768 769 if table and table._dbt and writelog and self.folder: 770 if isabs: 771 table._loggername = logfilename 772 else: 773 table._loggername = pjoin(self.folder, logfilename) 774 logfile = self.file_open(table._loggername, 'a') 775 logfile.write(message) 776 self.file_close(logfile)
777 778
779 - def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8', 780 credential_decoder=IDENTITY, driver_args={}, 781 adapter_args={},do_connect=True, after_connection=None):
782 self.db = db 783 self.dbengine = "None" 784 self.uri = uri 785 self.pool_size = pool_size 786 self.folder = folder 787 self.db_codec = db_codec 788 self._after_connection = after_connection 789 class Dummy(object): 790 lastrowid = 1 791 def __getattr__(self, value): 792 return lambda *a, **b: []
793 self.connection = Dummy() 794 self.cursor = Dummy() 795
796 - def sequence_name(self,tablename):
797 return '%s_sequence' % tablename
798
799 - def trigger_name(self,tablename):
800 return '%s_sequence' % tablename
801
802 - def varquote(self,name):
803 return name
804
805 - def create_table(self, table, 806 migrate=True, 807 fake_migrate=False, 808 polymodel=None):
809 db = table._db 810 fields = [] 811 # PostGIS geo fields are added after the table has been created 812 postcreation_fields = [] 813 sql_fields = {} 814 sql_fields_aux = {} 815 TFK = {} 816 tablename = table._tablename 817 sortable = 0 818 types = self.types 819 for field in table: 820 sortable += 1 821 field_name = field.name 822 field_type = field.type 823 if isinstance(field_type,SQLCustomType): 824 ftype = field_type.native or field_type.type 825 elif field_type.startswith('reference'): 826 referenced = field_type[10:].strip() 827 if referenced == '.': 828 referenced = tablename 829 constraint_name = self.constraint_name(tablename, field_name) 830 if not '.' in referenced \ 831 and referenced != tablename \ 832 and hasattr(table,'_primarykey'): 833 ftype = types['integer'] 834 else: 835 if hasattr(table,'_primarykey'): 836 rtablename,rfieldname = referenced.split('.') 837 rtable = db[rtablename] 838 rfield = rtable[rfieldname] 839 # must be PK reference or unique 840 if rfieldname in rtable._primarykey or \ 841 rfield.unique: 842 ftype = types[rfield.type[:9]] % \ 843 dict(length=rfield.length) 844 # multicolumn primary key reference? 845 if not rfield.unique and len(rtable._primarykey)>1: 846 # then it has to be a table level FK 847 if rtablename not in TFK: 848 TFK[rtablename] = {} 849 TFK[rtablename][rfieldname] = field_name 850 else: 851 ftype = ftype + \ 852 types['reference FK'] % dict( 853 constraint_name = constraint_name, # should be quoted 854 foreign_key = '%s (%s)' % (rtablename, 855 rfieldname), 856 table_name = tablename, 857 field_name = field_name, 858 on_delete_action=field.ondelete) 859 else: 860 # make a guess here for circular references 861 if referenced in db: 862 id_fieldname = db[referenced]._id.name 863 elif referenced == tablename: 864 id_fieldname = table._id.name 865 else: #make a guess 866 id_fieldname = 'id' 867 ftype = types[field_type[:9]] % dict( 868 index_name = field_name+'__idx', 869 field_name = field_name, 870 constraint_name = constraint_name, 871 foreign_key = '%s (%s)' % (referenced, 872 id_fieldname), 873 on_delete_action=field.ondelete) 874 elif field_type.startswith('list:reference'): 875 ftype = types[field_type[:14]] 876 elif field_type.startswith('decimal'): 877 precision, scale = map(int,field_type[8:-1].split(',')) 878 ftype = types[field_type[:7]] % \ 879 dict(precision=precision,scale=scale) 880 elif field_type.startswith('geo'): 881 if not hasattr(self,'srid'): 882 raise RuntimeError('Adapter does not support geometry') 883 srid = self.srid 884 geotype, parms = field_type[:-1].split('(') 885 if not geotype in types: 886 raise SyntaxError( 887 'Field: unknown field type: %s for %s' \ 888 % (field_type, field_name)) 889 ftype = types[geotype] 890 if self.dbengine == 'postgres' and geotype == 'geometry': 891 # parameters: schema, srid, dimension 892 dimension = 2 # GIS.dimension ??? 893 parms = parms.split(',') 894 if len(parms) == 3: 895 schema, srid, dimension = parms 896 elif len(parms) == 2: 897 schema, srid = parms 898 else: 899 schema = parms[0] 900 ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype] 901 ftype = ftype % dict(schema=schema, 902 tablename=tablename, 903 fieldname=field_name, srid=srid, 904 dimension=dimension) 905 postcreation_fields.append(ftype) 906 elif not field_type in types: 907 raise SyntaxError('Field: unknown field type: %s for %s' % \ 908 (field_type, field_name)) 909 else: 910 ftype = types[field_type]\ 911 % dict(length=field.length) 912 if not field_type.startswith('id') and \ 913 not field_type.startswith('reference'): 914 if field.notnull: 915 ftype += ' NOT NULL' 916 else: 917 ftype += self.ALLOW_NULL() 918 if field.unique: 919 ftype += ' UNIQUE' 920 if field.custom_qualifier: 921 ftype += ' %s' % field.custom_qualifier 922 923 # add to list of fields 924 sql_fields[field_name] = dict( 925 length=field.length, 926 unique=field.unique, 927 notnull=field.notnull, 928 sortable=sortable, 929 type=str(field_type), 930 sql=ftype) 931 932 if field.notnull and not field.default is None: 933 # Caveat: sql_fields and sql_fields_aux 934 # differ for default values. 935 # sql_fields is used to trigger migrations and sql_fields_aux 936 # is used for create tables. 937 # The reason is that we do not want to trigger 938 # a migration simply because a default value changes. 939 not_null = self.NOT_NULL(field.default, field_type) 940 ftype = ftype.replace('NOT NULL', not_null) 941 sql_fields_aux[field_name] = dict(sql=ftype) 942 # Postgres - PostGIS: 943 # geometry fields are added after the table has been created, not now 944 if not (self.dbengine == 'postgres' and \ 945 field_type.startswith('geom')): 946 fields.append('%s %s' % (field_name, ftype)) 947 other = ';' 948 949 # backend-specific extensions to fields 950 if self.dbengine == 'mysql': 951 if not hasattr(table, "_primarykey"): 952 fields.append('PRIMARY KEY(%s)' % table._id.name) 953 other = ' ENGINE=InnoDB CHARACTER SET utf8;' 954 955 fields = ',\n '.join(fields) 956 for rtablename in TFK: 957 rfields = TFK[rtablename] 958 pkeys = db[rtablename]._primarykey 959 fkeys = [ rfields[k] for k in pkeys ] 960 fields = fields + ',\n ' + \ 961 types['reference TFK'] % dict( 962 table_name = tablename, 963 field_name=', '.join(fkeys), 964 foreign_table = rtablename, 965 foreign_key = ', '.join(pkeys), 966 on_delete_action = field.ondelete) 967 968 if getattr(table,'_primarykey',None): 969 query = "CREATE TABLE %s(\n %s,\n %s) %s" % \ 970 (tablename, fields, 971 self.PRIMARY_KEY(', '.join(table._primarykey)),other) 972 else: 973 query = "CREATE TABLE %s(\n %s\n)%s" % \ 974 (tablename, fields, other) 975 976 if self.uri.startswith('sqlite:///') \ 977 or self.uri.startswith('spatialite:///'): 978 path_encoding = sys.getfilesystemencoding() \ 979 or locale.getdefaultlocale()[1] or 'utf8' 980 dbpath = self.uri[9:self.uri.rfind('/')]\ 981 .decode('utf8').encode(path_encoding) 982 else: 983 dbpath = self.folder 984 985 if not migrate: 986 return query 987 elif self.uri.startswith('sqlite:memory')\ 988 or self.uri.startswith('spatialite:memory'): 989 table._dbt = None 990 elif isinstance(migrate, str): 991 table._dbt = pjoin(dbpath, migrate) 992 else: 993 table._dbt = pjoin( 994 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename)) 995 996 if not table._dbt or not self.file_exists(table._dbt): 997 if table._dbt: 998 self.log('timestamp: %s\n%s\n' 999 % (datetime.datetime.today().isoformat(), 1000 query), table) 1001 if not fake_migrate: 1002 self.create_sequence_and_triggers(query,table) 1003 table._db.commit() 1004 # Postgres geom fields are added now, 1005 # after the table has been created 1006 for query in postcreation_fields: 1007 self.execute(query) 1008 table._db.commit() 1009 if table._dbt: 1010 tfile = self.file_open(table._dbt, 'w') 1011 pickle.dump(sql_fields, tfile) 1012 self.file_close(tfile) 1013 if fake_migrate: 1014 self.log('faked!\n', table) 1015 else: 1016 self.log('success!\n', table) 1017 else: 1018 tfile = self.file_open(table._dbt, 'r') 1019 try: 1020 sql_fields_old = pickle.load(tfile) 1021 except EOFError: 1022 self.file_close(tfile) 1023 raise RuntimeError('File %s appears corrupted' % table._dbt) 1024 self.file_close(tfile) 1025 if sql_fields != sql_fields_old: 1026 self.migrate_table(table, 1027 sql_fields, sql_fields_old, 1028 sql_fields_aux, None, 1029 fake_migrate=fake_migrate) 1030 return query
1031
1032 - def migrate_table( 1033 self, 1034 table, 1035 sql_fields, 1036 sql_fields_old, 1037 sql_fields_aux, 1038 logfile, 1039 fake_migrate=False, 1040 ):
1041 1042 # logfile is deprecated (moved to adapter.log method) 1043 db = table._db 1044 db._migrated.append(table._tablename) 1045 tablename = table._tablename 1046 def fix(item): 1047 k,v=item 1048 if not isinstance(v,dict): 1049 v=dict(type='unknown',sql=v) 1050 return k.lower(),v
1051 # make sure all field names are lower case to avoid 1052 # migrations because of case cahnge 1053 sql_fields = dict(map(fix,sql_fields.iteritems())) 1054 sql_fields_old = dict(map(fix,sql_fields_old.iteritems())) 1055 sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems())) 1056 if db._debug: 1057 logging.debug('migrating %s to %s' % (sql_fields_old,sql_fields)) 1058 1059 keys = sql_fields.keys() 1060 for key in sql_fields_old: 1061 if not key in keys: 1062 keys.append(key) 1063 new_add = self.concat_add(tablename) 1064 1065 metadata_change = False 1066 sql_fields_current = copy.copy(sql_fields_old) 1067 for key in keys: 1068 query = None 1069 if not key in sql_fields_old: 1070 sql_fields_current[key] = sql_fields[key] 1071 if self.dbengine in ('postgres',) and \ 1072 sql_fields[key]['type'].startswith('geometry'): 1073 # 'sql' == ftype in sql 1074 query = [ sql_fields[key]['sql'] ] 1075 else: 1076 query = ['ALTER TABLE %s ADD %s %s;' % \ 1077 (tablename, key, 1078 sql_fields_aux[key]['sql'].replace(', ', new_add))] 1079 metadata_change = True 1080 elif self.dbengine in ('sqlite', 'spatialite'): 1081 if key in sql_fields: 1082 sql_fields_current[key] = sql_fields[key] 1083 metadata_change = True 1084 elif not key in sql_fields: 1085 del sql_fields_current[key] 1086 ftype = sql_fields_old[key]['type'] 1087 if (self.dbengine in ('postgres',) and 1088 ftype.startswith('geometry')): 1089 geotype, parms = ftype[:-1].split('(') 1090 schema = parms.split(',')[0] 1091 query = [ "SELECT DropGeometryColumn ('%(schema)s', "+ 1092 "'%(table)s', '%(field)s');" % 1093 dict(schema=schema, table=tablename, field=key,) ] 1094 elif self.dbengine in ('firebird',): 1095 query = ['ALTER TABLE %s DROP %s;' % (tablename, key)] 1096 else: 1097 query = ['ALTER TABLE %s DROP COLUMN %s;' % 1098 (tablename, key)] 1099 metadata_change = True 1100 elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \ 1101 and not (key in table.fields and 1102 isinstance(table[key].type, SQLCustomType)) \ 1103 and not sql_fields[key]['type'].startswith('reference')\ 1104 and not sql_fields[key]['type'].startswith('double')\ 1105 and not sql_fields[key]['type'].startswith('id'): 1106 sql_fields_current[key] = sql_fields[key] 1107 t = tablename 1108 tt = sql_fields_aux[key]['sql'].replace(', ', new_add) 1109 if self.dbengine in ('firebird',): 1110 drop_expr = 'ALTER TABLE %s DROP %s;' 1111 else: 1112 drop_expr = 'ALTER TABLE %s DROP COLUMN %s;' 1113 key_tmp = key + '__tmp' 1114 query = ['ALTER TABLE %s ADD %s %s;' % (t, key_tmp, tt), 1115 'UPDATE %s SET %s=%s;' % (t, key_tmp, key), 1116 drop_expr % (t, key), 1117 'ALTER TABLE %s ADD %s %s;' % (t, key, tt), 1118 'UPDATE %s SET %s=%s;' % (t, key, key_tmp), 1119 drop_expr % (t, key_tmp)] 1120 metadata_change = True 1121 elif sql_fields[key]['type'] != sql_fields_old[key]['type']: 1122 sql_fields_current[key] = sql_fields[key] 1123 metadata_change = True 1124 1125 if query: 1126 self.log('timestamp: %s\n' 1127 % datetime.datetime.today().isoformat(), table) 1128 db['_lastsql'] = '\n'.join(query) 1129 for sub_query in query: 1130 self.log(sub_query + '\n', table) 1131 if fake_migrate: 1132 if db._adapter.commit_on_alter_table: 1133 self.save_dbt(table,sql_fields_current) 1134 self.log('faked!\n', table) 1135 else: 1136 self.execute(sub_query) 1137 # Caveat: mysql, oracle and firebird 1138 # do not allow multiple alter table 1139 # in one transaction so we must commit 1140 # partial transactions and 1141 # update table._dbt after alter table. 1142 if db._adapter.commit_on_alter_table: 1143 db.commit() 1144 self.save_dbt(table,sql_fields_current) 1145 self.log('success!\n', table) 1146 1147 elif metadata_change: 1148 self.save_dbt(table,sql_fields_current) 1149 1150 if metadata_change and not (query and db._adapter.commit_on_alter_table): 1151 db.commit() 1152 self.save_dbt(table,sql_fields_current) 1153 self.log('success!\n', table) 1154
1155 - def save_dbt(self,table, sql_fields_current):
1156 tfile = self.file_open(table._dbt, 'w') 1157 pickle.dump(sql_fields_current, tfile) 1158 self.file_close(tfile)
1159
1160 - def LOWER(self, first):
1161 return 'LOWER(%s)' % self.expand(first)
1162
1163 - def UPPER(self, first):
1164 return 'UPPER(%s)' % self.expand(first)
1165
1166 - def COUNT(self, first, distinct=None):
1167 return ('COUNT(%s)' if not distinct else 'COUNT(DISTINCT %s)') \ 1168 % self.expand(first)
1169
1170 - def EXTRACT(self, first, what):
1171 return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
1172
1173 - def EPOCH(self, first):
1174 return self.EXTRACT(first, 'epoch')
1175
1176 - def LENGTH(self, first):
1177 return "LENGTH(%s)" % self.expand(first)
1178
1179 - def AGGREGATE(self, first, what):
1180 return "%s(%s)" % (what, self.expand(first))
1181
1182 - def JOIN(self):
1183 return 'JOIN'
1184
1185 - def LEFT_JOIN(self):
1186 return 'LEFT JOIN'
1187
1188 - def RANDOM(self):
1189 return 'Random()'
1190
1191 - def NOT_NULL(self, default, field_type):
1192 return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
1193
1194 - def COALESCE(self, first, second):
1195 expressions = [self.expand(first)]+[self.expand(e) for e in second] 1196 return 'COALESCE(%s)' % ','.join(expressions)
1197
1198 - def COALESCE_ZERO(self, first):
1199 return 'COALESCE(%s,0)' % self.expand(first)
1200
1201 - def RAW(self, first):
1202 return first
1203
1204 - def ALLOW_NULL(self):
1205 return ''
1206
1207 - def SUBSTRING(self, field, parameters):
1208 return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
1209
1210 - def PRIMARY_KEY(self, key):
1211 return 'PRIMARY KEY(%s)' % key
1212
1213 - def _drop(self, table, mode):
1214 return ['DROP TABLE %s;' % table]
1215
1216 - def drop(self, table, mode=''):
1217 db = table._db 1218 queries = self._drop(table, mode) 1219 for query in queries: 1220 if table._dbt: 1221 self.log(query + '\n', table) 1222 self.execute(query) 1223 db.commit() 1224 del db[table._tablename] 1225 del db.tables[db.tables.index(table._tablename)] 1226 db._remove_references_to(table) 1227 if table._dbt: 1228 self.file_delete(table._dbt) 1229 self.log('success!\n', table)
1230
1231 - def _insert(self, table, fields):
1232 if fields: 1233 keys = ','.join(f.name for f, v in fields) 1234 values = ','.join(self.expand(v, f.type) for f, v in fields) 1235 return 'INSERT INTO %s(%s) VALUES (%s);' % (table, keys, values) 1236 else: 1237 return self._insert_empty(table)
1238
1239 - def _insert_empty(self, table):
1240 return 'INSERT INTO %s DEFAULT VALUES;' % table
1241
1242 - def insert(self, table, fields):
1243 query = self._insert(table,fields) 1244 try: 1245 self.execute(query) 1246 except Exception: 1247 e = sys.exc_info()[1] 1248 if hasattr(table,'_on_insert_error'): 1249 return table._on_insert_error(table,fields,e) 1250 raise e 1251 if hasattr(table,'_primarykey'): 1252 return dict([(k[0].name, k[1]) for k in fields \ 1253 if k[0].name in table._primarykey]) 1254 id = self.lastrowid(table) 1255 if not isinstance(id,int): 1256 return id 1257 rid = Reference(id) 1258 (rid._table, rid._record) = (table, None) 1259 return rid
1260
1261 - def bulk_insert(self, table, items):
1262 return [self.insert(table,item) for item in items]
1263
1264 - def NOT(self, first):
1265 return '(NOT %s)' % self.expand(first)
1266
1267 - def AND(self, first, second):
1268 return '(%s AND %s)' % (self.expand(first), self.expand(second))
1269
1270 - def OR(self, first, second):
1271 return '(%s OR %s)' % (self.expand(first), self.expand(second))
1272
1273 - def BELONGS(self, first, second):
1274 if isinstance(second, str): 1275 return '(%s IN (%s))' % (self.expand(first), second[:-1]) 1276 if not second: 1277 return '(1=0)' 1278 items = ','.join(self.expand(item, first.type) for item in second) 1279 return '(%s IN (%s))' % (self.expand(first), items)
1280
1281 - def REGEXP(self, first, second):
1282 "regular expression operator" 1283 raise NotImplementedError
1284
1285 - def LIKE(self, first, second):
1286 "case sensitive like operator" 1287 raise NotImplementedError
1288
1289 - def ILIKE(self, first, second):
1290 "case in-sensitive like operator" 1291 return '(%s LIKE %s)' % (self.expand(first), 1292 self.expand(second, 'string'))
1293
1294 - def STARTSWITH(self, first, second):
1295 return '(%s LIKE %s)' % (self.expand(first), 1296 self.expand(second+'%', 'string'))
1297
1298 - def ENDSWITH(self, first, second):
1299 return '(%s LIKE %s)' % (self.expand(first), 1300 self.expand('%'+second, 'string'))
1301
1302 - def CONTAINS(self,first,second,case_sensitive=False):
1303 if first.type in ('string','text', 'json'): 1304 if isinstance(second,Expression): 1305 second = Expression(None,self.CONCAT('%',Expression( 1306 None,self.REPLACE(second,('%','%%'))),'%')) 1307 else: 1308 second = '%'+str(second).replace('%','%%')+'%' 1309 elif first.type.startswith('list:'): 1310 if isinstance(second,Expression): 1311 second = Expression(None,self.CONCAT( 1312 '%|',Expression(None,self.REPLACE( 1313 Expression(None,self.REPLACE( 1314 second,('%','%%'))),('|','||'))),'|%')) 1315 else: 1316 second = '%|'+str(second).replace('%','%%')\ 1317 .replace('|','||')+'|%' 1318 op = case_sensitive and self.LIKE or self.ILIKE 1319 return op(first,second)
1320
1321 - def EQ(self, first, second=None):
1322 if second is None: 1323 return '(%s IS NULL)' % self.expand(first) 1324 return '(%s = %s)' % (self.expand(first), 1325 self.expand(second, first.type))
1326
1327 - def NE(self, first, second=None):
1328 if second is None: 1329 return '(%s IS NOT NULL)' % self.expand(first) 1330 return '(%s <> %s)' % (self.expand(first), 1331 self.expand(second, first.type))
1332
1333 - def LT(self,first,second=None):
1334 if second is None: 1335 raise RuntimeError("Cannot compare %s < None" % first) 1336 return '(%s < %s)' % (self.expand(first), 1337 self.expand(second,first.type))
1338
1339 - def LE(self,first,second=None):
1340 if second is None: 1341 raise RuntimeError("Cannot compare %s <= None" % first) 1342 return '(%s <= %s)' % (self.expand(first), 1343 self.expand(second,first.type))
1344
1345 - def GT(self,first,second=None):
1346 if second is None: 1347 raise RuntimeError("Cannot compare %s > None" % first) 1348 return '(%s > %s)' % (self.expand(first), 1349 self.expand(second,first.type))
1350
1351 - def GE(self,first,second=None):
1352 if second is None: 1353 raise RuntimeError("Cannot compare %s >= None" % first) 1354 return '(%s >= %s)' % (self.expand(first), 1355 self.expand(second,first.type))
1356
1357 - def is_numerical_type(self, ftype):
1358 return ftype in ('integer','boolean','double','bigint') or \ 1359 ftype.startswith('decimal')
1360
1361 - def REPLACE(self, first, (second, third)):
1362 return 'REPLACE(%s,%s,%s)' % (self.expand(first,'string'), 1363 self.expand(second,'string'), 1364 self.expand(third,'string'))
1365
1366 - def CONCAT(self, *items):
1367 return '(%s)' % ' || '.join(self.expand(x,'string') for x in items)
1368
1369 - def ADD(self, first, second):
1370 if self.is_numerical_type(first.type): 1371 return '(%s + %s)' % (self.expand(first), 1372 self.expand(second, first.type)) 1373 else: 1374 return self.CONCAT(first, second)
1375
1376 - def SUB(self, first, second):
1377 return '(%s - %s)' % (self.expand(first), 1378 self.expand(second, first.type))
1379
1380 - def MUL(self, first, second):
1381 return '(%s * %s)' % (self.expand(first), 1382 self.expand(second, first.type))
1383
1384 - def DIV(self, first, second):
1385 return '(%s / %s)' % (self.expand(first), 1386 self.expand(second, first.type))
1387
1388 - def MOD(self, first, second):
1389 return '(%s %% %s)' % (self.expand(first), 1390 self.expand(second, first.type))
1391
1392 - def AS(self, first, second):
1393 return '%s AS %s' % (self.expand(first), second)
1394
1395 - def ON(self, first, second):
1396 if use_common_filters(second): 1397 second = self.common_filter(second,[first._tablename]) 1398 return '%s ON %s' % (self.expand(first), self.expand(second))
1399
1400 - def INVERT(self, first):
1401 return '%s DESC' % self.expand(first)
1402
1403 - def COMMA(self, first, second):
1404 return '%s, %s' % (self.expand(first), self.expand(second))
1405
1406 - def CAST(self, first, second):
1407 return 'CAST(%s AS %s)' % (first, second)
1408
1409 - def expand(self, expression, field_type=None):
1410 if isinstance(expression, Field): 1411 out = '%s.%s' % (expression.table._tablename, expression.name) 1412 if field_type == 'string' and not expression.type in ( 1413 'string','text','json','password'): 1414 out = self.CAST(out, self.types['text']) 1415 return out 1416 elif isinstance(expression, (Expression, Query)): 1417 first = expression.first 1418 second = expression.second 1419 op = expression.op 1420 optional_args = expression.optional_args or {} 1421 if not second is None: 1422 out = op(first, second, **optional_args) 1423 elif not first is None: 1424 out = op(first,**optional_args) 1425 elif isinstance(op, str): 1426 if op.endswith(';'): 1427 op=op[:-1] 1428 out = '(%s)' % op 1429 else: 1430 out = op() 1431 return out 1432 elif field_type: 1433 return str(self.represent(expression,field_type)) 1434 elif isinstance(expression,(list,tuple)): 1435 return ','.join(self.represent(item,field_type) \ 1436 for item in expression) 1437 elif isinstance(expression, bool): 1438 return '1' if expression else '0' 1439 else: 1440 return str(expression)
1441
1442 - def table_alias(self,name):
1443 return str(name if isinstance(name,Table) else self.db[name])
1444
1445 - def alias(self, table, alias):
1446 """ 1447 Given a table object, makes a new table object 1448 with alias name. 1449 """ 1450 other = copy.copy(table) 1451 other['_ot'] = other._ot or other._tablename 1452 other['ALL'] = SQLALL(other) 1453 other['_tablename'] = alias 1454 for fieldname in other.fields: 1455 other[fieldname] = copy.copy(other[fieldname]) 1456 other[fieldname]._tablename = alias 1457 other[fieldname].tablename = alias 1458 other[fieldname].table = other 1459 table._db[alias] = other 1460 return other
1461
1462 - def _truncate(self, table, mode=''):
1463 tablename = table._tablename 1464 return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
1465
1466 - def truncate(self, table, mode= ' '):
1467 # Prepare functions "write_to_logfile" and "close_logfile" 1468 try: 1469 queries = table._db._adapter._truncate(table, mode) 1470 for query in queries: 1471 self.log(query + '\n', table) 1472 self.execute(query) 1473 table._db.commit() 1474 self.log('success!\n', table) 1475 finally: 1476 pass
1477
1478 - def _update(self, tablename, query, fields):
1479 if query: 1480 if use_common_filters(query): 1481 query = self.common_filter(query, [tablename]) 1482 sql_w = ' WHERE ' + self.expand(query) 1483 else: 1484 sql_w = '' 1485 sql_v = ','.join(['%s=%s' % (field.name, 1486 self.expand(value, field.type)) \ 1487 for (field, value) in fields]) 1488 tablename = "%s" % self.db[tablename] 1489 return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
1490
1491 - def update(self, tablename, query, fields):
1492 sql = self._update(tablename, query, fields) 1493 try: 1494 self.execute(sql) 1495 except Exception: 1496 e = sys.exc_info()[1] 1497 table = self.db[tablename] 1498 if hasattr(table,'_on_update_error'): 1499 return table._on_update_error(table,query,fields,e) 1500 raise e 1501 try: 1502 return self.cursor.rowcount 1503 except: 1504 return None
1505
1506 - def _delete(self, tablename, query):
1507 if query: 1508 if use_common_filters(query): 1509 query = self.common_filter(query, [tablename]) 1510 sql_w = ' WHERE ' + self.expand(query) 1511 else: 1512 sql_w = '' 1513 return 'DELETE FROM %s%s;' % (tablename, sql_w)
1514
1515 - def delete(self, tablename, query):
1516 sql = self._delete(tablename, query) 1517 ### special code to handle CASCADE in SQLite & SpatiaLite 1518 db = self.db 1519 table = db[tablename] 1520 if self.dbengine in ('sqlite', 'spatialite') and table._referenced_by: 1521 deleted = [x[table._id.name] for x in db(query).select(table._id)] 1522 ### end special code to handle CASCADE in SQLite & SpatiaLite 1523 self.execute(sql) 1524 try: 1525 counter = self.cursor.rowcount 1526 except: 1527 counter = None 1528 ### special code to handle CASCADE in SQLite & SpatiaLite 1529 if self.dbengine in ('sqlite', 'spatialite') and counter: 1530 for field in table._referenced_by: 1531 if field.type=='reference '+table._tablename \ 1532 and field.ondelete=='CASCADE': 1533 db(field.belongs(deleted)).delete() 1534 ### end special code to handle CASCADE in SQLite & SpatiaLite 1535 return counter
1536
1537 - def get_table(self, query):
1538 tablenames = self.tables(query) 1539 if len(tablenames)==1: 1540 return tablenames[0] 1541 elif len(tablenames)<1: 1542 raise RuntimeError("No table selected") 1543 else: 1544 raise RuntimeError("Too many tables selected")
1545
1546 - def expand_all(self, fields, tablenames):
1547 db = self.db 1548 new_fields = [] 1549 append = new_fields.append 1550 for item in fields: 1551 if isinstance(item,SQLALL): 1552 new_fields += item._table 1553 elif isinstance(item,str): 1554 if REGEX_TABLE_DOT_FIELD.match(item): 1555 tablename,fieldname = item.split('.') 1556 append(db[tablename][fieldname]) 1557 else: 1558 append(Expression(db,lambda item=item:item)) 1559 else: 1560 append(item) 1561 # ## if no fields specified take them all from the requested tables 1562 if not new_fields: 1563 for table in tablenames: 1564 for field in db[table]: 1565 append(field) 1566 return new_fields
1567
1568 - def _select(self, query, fields, attributes):
1569 tables = self.tables 1570 for key in set(attributes.keys())-SELECT_ARGS: 1571 raise SyntaxError('invalid select attribute: %s' % key) 1572 args_get = attributes.get 1573 tablenames = tables(query) 1574 tablenames_for_common_filters = tablenames 1575 for field in fields: 1576 if isinstance(field, basestring) \ 1577 and REGEX_TABLE_DOT_FIELD.match(field): 1578 tn,fn = field.split('.') 1579 field = self.db[tn][fn] 1580 for tablename in tables(field): 1581 if not tablename in tablenames: 1582 tablenames.append(tablename) 1583 1584 if len(tablenames) < 1: 1585 raise SyntaxError('Set: no tables selected') 1586 self._colnames = map(self.expand, fields) 1587 def geoexpand(field): 1588 if isinstance(field.type,str) and field.type.startswith('geometry'): 1589 field = field.st_astext() 1590 return self.expand(field)
1591 sql_f = ', '.join(map(geoexpand, fields)) 1592 sql_o = '' 1593 sql_s = '' 1594 left = args_get('left', False) 1595 inner_join = args_get('join', False) 1596 distinct = args_get('distinct', False) 1597 groupby = args_get('groupby', False) 1598 orderby = args_get('orderby', False) 1599 having = args_get('having', False) 1600 limitby = args_get('limitby', False) 1601 orderby_on_limitby = args_get('orderby_on_limitby', True) 1602 for_update = args_get('for_update', False) 1603 if self.can_select_for_update is False and for_update is True: 1604 raise SyntaxError('invalid select attribute: for_update') 1605 if distinct is True: 1606 sql_s += 'DISTINCT' 1607 elif distinct: 1608 sql_s += 'DISTINCT ON (%s)' % distinct 1609 if inner_join: 1610 icommand = self.JOIN() 1611 if not isinstance(inner_join, (tuple, list)): 1612 inner_join = [inner_join] 1613 ijoint = [t._tablename for t in inner_join 1614 if not isinstance(t,Expression)] 1615 ijoinon = [t for t in inner_join if isinstance(t, Expression)] 1616 itables_to_merge={} #issue 490 1617 [itables_to_merge.update( 1618 dict.fromkeys(tables(t))) for t in ijoinon] 1619 ijoinont = [t.first._tablename for t in ijoinon] 1620 [itables_to_merge.pop(t) for t in ijoinont 1621 if t in itables_to_merge] #issue 490 1622 iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys() 1623 iexcluded = [t for t in tablenames 1624 if not t in iimportant_tablenames] 1625 if left: 1626 join = attributes['left'] 1627 command = self.LEFT_JOIN() 1628 if not isinstance(join, (tuple, list)): 1629 join = [join] 1630 joint = [t._tablename for t in join 1631 if not isinstance(t, Expression)] 1632 joinon = [t for t in join if isinstance(t, Expression)] 1633 #patch join+left patch (solves problem with ordering in left joins) 1634 tables_to_merge={} 1635 [tables_to_merge.update( 1636 dict.fromkeys(tables(t))) for t in joinon] 1637 joinont = [t.first._tablename for t in joinon] 1638 [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge] 1639 tablenames_for_common_filters = [t for t in tablenames 1640 if not t in joinont ] 1641 important_tablenames = joint + joinont + tables_to_merge.keys() 1642 excluded = [t for t in tablenames 1643 if not t in important_tablenames ] 1644 else: 1645 excluded = tablenames 1646 1647 if use_common_filters(query): 1648 query = self.common_filter(query,tablenames_for_common_filters) 1649 sql_w = ' WHERE ' + self.expand(query) if query else '' 1650 1651 if inner_join and not left: 1652 sql_t = ', '.join([self.table_alias(t) for t in iexcluded + \ 1653 itables_to_merge.keys()]) 1654 for t in ijoinon: 1655 sql_t += ' %s %s' % (icommand, t) 1656 elif not inner_join and left: 1657 sql_t = ', '.join([self.table_alias(t) for t in excluded + \ 1658 tables_to_merge.keys()]) 1659 if joint: 1660 sql_t += ' %s %s' % (command, 1661 ','.join([self.table_alias(t) for t in joint])) 1662 for t in joinon: 1663 sql_t += ' %s %s' % (command, t) 1664 elif inner_join and left: 1665 all_tables_in_query = set(important_tablenames + \ 1666 iimportant_tablenames + \ 1667 tablenames) 1668 tables_in_joinon = set(joinont + ijoinont) 1669 tables_not_in_joinon = \ 1670 all_tables_in_query.difference(tables_in_joinon) 1671 sql_t = ','.join([self.table_alias(t) for t in tables_not_in_joinon]) 1672 for t in ijoinon: 1673 sql_t += ' %s %s' % (icommand, t) 1674 if joint: 1675 sql_t += ' %s %s' % (command, 1676 ','.join([self.table_alias(t) for t in joint])) 1677 for t in joinon: 1678 sql_t += ' %s %s' % (command, t) 1679 else: 1680 sql_t = ', '.join(self.table_alias(t) for t in tablenames) 1681 if groupby: 1682 if isinstance(groupby, (list, tuple)): 1683 groupby = xorify(groupby) 1684 sql_o += ' GROUP BY %s' % self.expand(groupby) 1685 if having: 1686 sql_o += ' HAVING %s' % attributes['having'] 1687 if orderby: 1688 if isinstance(orderby, (list, tuple)): 1689 orderby = xorify(orderby) 1690 if str(orderby) == '<random>': 1691 sql_o += ' ORDER BY %s' % self.RANDOM() 1692 else: 1693 sql_o += ' ORDER BY %s' % self.expand(orderby) 1694 if (limitby and not groupby and tablenames and orderby_on_limitby and not orderby): 1695 sql_o += ' ORDER BY %s' % ', '.join(['%s.%s'%(t,x) for t in tablenames for x in (hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey or [self.db[t]._id.name])]) 1696 # oracle does not support limitby 1697 sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby) 1698 if for_update and self.can_select_for_update is True: 1699 sql = sql.rstrip(';') + ' FOR UPDATE;' 1700 return sql 1701
1702 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
1703 if limitby: 1704 (lmin, lmax) = limitby 1705 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin) 1706 return 'SELECT %s %s FROM %s%s%s;' % \ 1707 (sql_s, sql_f, sql_t, sql_w, sql_o)
1708
1709 - def _fetchall(self):
1710 return self.cursor.fetchall()
1711
1712 - def _select_aux(self,sql,fields,attributes):
1713 args_get = attributes.get 1714 cache = args_get('cache',None) 1715 if not cache: 1716 self.execute(sql) 1717 rows = self._fetchall() 1718 else: 1719 (cache_model, time_expire) = cache 1720 key = self.uri + '/' + sql + '/rows' 1721 if len(key)>200: key = hashlib_md5(key).hexdigest() 1722 def _select_aux2(): 1723 self.execute(sql) 1724 return self._fetchall()
1725 rows = cache_model(key,_select_aux2,time_expire) 1726 if isinstance(rows,tuple): 1727 rows = list(rows) 1728 limitby = args_get('limitby', None) or (0,) 1729 rows = self.rowslice(rows,limitby[0],None) 1730 processor = args_get('processor',self.parse) 1731 cacheable = args_get('cacheable',False) 1732 return processor(rows,fields,self._colnames,cacheable=cacheable) 1733
1734 - def select(self, query, fields, attributes):
1735 """ 1736 Always returns a Rows object, possibly empty. 1737 """ 1738 sql = self._select(query, fields, attributes) 1739 cache = attributes.get('cache', None) 1740 if cache and attributes.get('cacheable',False): 1741 del attributes['cache'] 1742 (cache_model, time_expire) = cache 1743 key = self.uri + '/' + sql 1744 if len(key)>200: key = hashlib_md5(key).hexdigest() 1745 args = (sql,fields,attributes) 1746 return cache_model( 1747 key, 1748 lambda self=self,args=args:self._select_aux(*args), 1749 time_expire) 1750 else: 1751 return self._select_aux(sql,fields,attributes)
1752
1753 - def _count(self, query, distinct=None):
1754 tablenames = self.tables(query) 1755 if query: 1756 if use_common_filters(query): 1757 query = self.common_filter(query, tablenames) 1758 sql_w = ' WHERE ' + self.expand(query) 1759 else: 1760 sql_w = '' 1761 sql_t = ','.join(self.table_alias(t) for t in tablenames) 1762 if distinct: 1763 if isinstance(distinct,(list, tuple)): 1764 distinct = xorify(distinct) 1765 sql_d = self.expand(distinct) 1766 return 'SELECT count(DISTINCT %s) FROM %s%s;' % \ 1767 (sql_d, sql_t, sql_w) 1768 return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
1769
1770 - def count(self, query, distinct=None):
1771 self.execute(self._count(query, distinct)) 1772 return self.cursor.fetchone()[0]
1773
1774 - def tables(self, *queries):
1775 tables = set() 1776 for query in queries: 1777 if isinstance(query, Field): 1778 tables.add(query.tablename) 1779 elif isinstance(query, (Expression, Query)): 1780 if not query.first is None: 1781 tables = tables.union(self.tables(query.first)) 1782 if not query.second is None: 1783 tables = tables.union(self.tables(query.second)) 1784 return list(tables)
1785
1786 - def commit(self):
1787 if self.connection: 1788 return self.connection.commit()
1789
1790 - def rollback(self):
1791 if self.connection: 1792 return self.connection.rollback()
1793
1794 - def close_connection(self):
1795 if self.connection: 1796 r = self.connection.close() 1797 self.connection = None 1798 return r
1799
1800 - def distributed_transaction_begin(self, key):
1801 return
1802
1803 - def prepare(self, key):
1804 if self.connection: self.connection.prepare()
1805
1806 - def commit_prepared(self, key):
1807 if self.connection: self.connection.commit()
1808
1809 - def rollback_prepared(self, key):
1810 if self.connection: self.connection.rollback()
1811
1812 - def concat_add(self, tablename):
1813 return ', ADD '
1814
1815 - def constraint_name(self, table, fieldname):
1816 return '%s_%s__constraint' % (table,fieldname)
1817
1818 - def create_sequence_and_triggers(self, query, table, **args):
1819 self.execute(query)
1820
1821 - def log_execute(self, *a, **b):
1822 if not self.connection: return None 1823 command = a[0] 1824 if hasattr(self,'filter_sql_command'): 1825 command = self.filter_sql_command(command) 1826 if self.db._debug: 1827 LOGGER.debug('SQL: %s' % command) 1828 self.db._lastsql = command 1829 t0 = time.time() 1830 ret = self.cursor.execute(command, *a[1:], **b) 1831 self.db._timings.append((command,time.time()-t0)) 1832 del self.db._timings[:-TIMINGSSIZE] 1833 return ret
1834
1835 - def execute(self, *a, **b):
1836 return self.log_execute(*a, **b)
1837
1838 - def represent(self, obj, fieldtype):
1839 field_is_type = fieldtype.startswith 1840 if isinstance(obj, CALLABLETYPES): 1841 obj = obj() 1842 if isinstance(fieldtype, SQLCustomType): 1843 value = fieldtype.encoder(obj) 1844 if fieldtype.type in ('string','text', 'json'): 1845 return self.adapt(value) 1846 return value 1847 if isinstance(obj, (Expression, Field)): 1848 return str(obj) 1849 if field_is_type('list:'): 1850 if not obj: 1851 obj = [] 1852 elif not isinstance(obj, (list, tuple)): 1853 obj = [obj] 1854 if field_is_type('list:string'): 1855 obj = map(str,obj) 1856 else: 1857 obj = map(int,[o for o in obj if o != '']) 1858 # we don't want to bar_encode json objects 1859 if isinstance(obj, (list, tuple)) and (not fieldtype == "json"): 1860 obj = bar_encode(obj) 1861 if obj is None: 1862 return 'NULL' 1863 if obj == '' and not fieldtype[:2] in ['st', 'te', 'js', 'pa', 'up']: 1864 return 'NULL' 1865 r = self.represent_exceptions(obj, fieldtype) 1866 if not r is None: 1867 return r 1868 if fieldtype == 'boolean': 1869 if obj and not str(obj)[:1].upper() in '0F': 1870 return self.smart_adapt(self.TRUE) 1871 else: 1872 return self.smart_adapt(self.FALSE) 1873 if fieldtype == 'id' or fieldtype == 'integer': 1874 return str(long(obj)) 1875 if field_is_type('decimal'): 1876 return str(obj) 1877 elif field_is_type('reference'): # reference 1878 if fieldtype.find('.')>0: 1879 return repr(obj) 1880 elif isinstance(obj, (Row, Reference)): 1881 return str(obj['id']) 1882 return str(long(obj)) 1883 elif fieldtype == 'double': 1884 return repr(float(obj)) 1885 if isinstance(obj, unicode): 1886 obj = obj.encode(self.db_codec) 1887 if fieldtype == 'blob': 1888 obj = base64.b64encode(str(obj)) 1889 elif fieldtype == 'date': 1890 if isinstance(obj, (datetime.date, datetime.datetime)): 1891 obj = obj.isoformat()[:10] 1892 else: 1893 obj = str(obj) 1894 elif fieldtype == 'datetime': 1895 if isinstance(obj, datetime.datetime): 1896 obj = obj.isoformat(self.T_SEP)[:19] 1897 elif isinstance(obj, datetime.date): 1898 obj = obj.isoformat()[:10]+self.T_SEP+'00:00:00' 1899 else: 1900 obj = str(obj) 1901 elif fieldtype == 'time': 1902 if isinstance(obj, datetime.time): 1903 obj = obj.isoformat()[:10] 1904 else: 1905 obj = str(obj) 1906 elif fieldtype == 'json': 1907 if not self.native_json: 1908 if have_serializers: 1909 obj = serializers.json(obj) 1910 elif simplejson: 1911 obj = simplejson.dumps(obj) 1912 else: 1913 raise RuntimeError("missing simplejson") 1914 if not isinstance(obj,bytes): 1915 obj = bytes(obj) 1916 try: 1917 obj.decode(self.db_codec) 1918 except: 1919 obj = obj.decode('latin1').encode(self.db_codec) 1920 return self.adapt(obj)
1921
1922 - def represent_exceptions(self, obj, fieldtype):
1923 return None
1924
1925 - def lastrowid(self, table):
1926 return None
1927
1928 - def rowslice(self, rows, minimum=0, maximum=None):
1929 """ 1930 By default this function does nothing; 1931 overload when db does not do slicing. 1932 """ 1933 return rows
1934
1935 - def parse_value(self, value, field_type, blob_decode=True):
1936 if field_type != 'blob' and isinstance(value, str): 1937 try: 1938 value = value.decode(self.db._db_codec) 1939 except Exception: 1940 pass 1941 if isinstance(value, unicode): 1942 value = value.encode('utf-8') 1943 if isinstance(field_type, SQLCustomType): 1944 value = field_type.decoder(value) 1945 if not isinstance(field_type, str) or value is None: 1946 return value 1947 elif field_type in ('string', 'text', 'password', 'upload', 'dict'): 1948 return value 1949 elif field_type.startswith('geo'): 1950 return value 1951 elif field_type == 'blob' and not blob_decode: 1952 return value 1953 else: 1954 key = REGEX_TYPE.match(field_type).group(0) 1955 return self.parsemap[key](value,field_type)
1956
1957 - def parse_reference(self, value, field_type):
1958 referee = field_type[10:].strip() 1959 if not '.' in referee: 1960 value = Reference(value) 1961 value._table, value._record = self.db[referee], None 1962 return value
1963
1964 - def parse_boolean(self, value, field_type):
1965 return value == self.TRUE or str(value)[:1].lower() == 't'
1966
1967 - def parse_date(self, value, field_type):
1968 if isinstance(value, datetime.datetime): 1969 return value.date() 1970 if not isinstance(value, (datetime.date,datetime.datetime)): 1971 (y, m, d) = map(int, str(value)[:10].strip().split('-')) 1972 value = datetime.date(y, m, d) 1973 return value
1974
1975 - def parse_time(self, value, field_type):
1976 if not isinstance(value, datetime.time): 1977 time_items = map(int,str(value)[:8].strip().split(':')[:3]) 1978 if len(time_items) == 3: 1979 (h, mi, s) = time_items 1980 else: 1981 (h, mi, s) = time_items + [0] 1982 value = datetime.time(h, mi, s) 1983 return value
1984
1985 - def parse_datetime(self, value, field_type):
1986 if not isinstance(value, datetime.datetime): 1987 value = str(value) 1988 date_part,time_part,timezone = value[:10],value[11:19],value[19:] 1989 if '+' in timezone: 1990 ms,tz = timezone.split('+') 1991 h,m = tz.split(':') 1992 dt = datetime.timedelta(seconds=3600*int(h)+60*int(m)) 1993 elif '-' in timezone: 1994 ms,tz = timezone.split('-') 1995 h,m = tz.split(':') 1996 dt = -datetime.timedelta(seconds=3600*int(h)+60*int(m)) 1997 else: 1998 dt = None 1999 (y, m, d) = map(int,date_part.split('-')) 2000 time_parts = time_part and time_part.split(':')[:3] or (0,0,0) 2001 while len(time_parts)<3: time_parts.append(0) 2002 time_items = map(int,time_parts) 2003 (h, mi, s) = time_items 2004 value = datetime.datetime(y, m, d, h, mi, s) 2005 if dt: 2006 value = value + dt 2007 return value
2008
2009 - def parse_blob(self, value, field_type):
2010 return base64.b64decode(str(value))
2011
2012 - def parse_decimal(self, value, field_type):
2013 decimals = int(field_type[8:-1].split(',')[-1]) 2014 if self.dbengine in ('sqlite', 'spatialite'): 2015 value = ('%.' + str(decimals) + 'f') % value 2016 if not isinstance(value, decimal.Decimal): 2017 value = decimal.Decimal(str(value)) 2018 return value
2019
2020 - def parse_list_integers(self, value, field_type):
2021 if not isinstance(self, NoSQLAdapter): 2022 value = bar_decode_integer(value) 2023 return value
2024
2025 - def parse_list_references(self, value, field_type):
2026 if not isinstance(self, NoSQLAdapter): 2027 value = bar_decode_integer(value) 2028 return [self.parse_reference(r, field_type[5:]) for r in value]
2029
2030 - def parse_list_strings(self, value, field_type):
2031 if not isinstance(self, NoSQLAdapter): 2032 value = bar_decode_string(value) 2033 return value
2034
2035 - def parse_id(self, value, field_type):
2036 return long(value)
2037
2038 - def parse_integer(self, value, field_type):
2039 return long(value)
2040
2041 - def parse_double(self, value, field_type):
2042 return float(value)
2043
2044 - def parse_json(self, value, field_type):
2045 if not self.native_json: 2046 if not isinstance(value, basestring): 2047 raise RuntimeError('json data not a string') 2048 if isinstance(value, unicode): 2049 value = value.encode('utf-8') 2050 if have_serializers: 2051 value = serializers.loads_json(value) 2052 elif simplejson: 2053 value = simplejson.loads(value) 2054 else: 2055 raise RuntimeError("missing simplejson") 2056 return value
2057
2058 - def build_parsemap(self):
2059 self.parsemap = { 2060 'id':self.parse_id, 2061 'integer':self.parse_integer, 2062 'bigint':self.parse_integer, 2063 'float':self.parse_double, 2064 'double':self.parse_double, 2065 'reference':self.parse_reference, 2066 'boolean':self.parse_boolean, 2067 'date':self.parse_date, 2068 'time':self.parse_time, 2069 'datetime':self.parse_datetime, 2070 'blob':self.parse_blob, 2071 'decimal':self.parse_decimal, 2072 'json':self.parse_json, 2073 'list:integer':self.parse_list_integers, 2074 'list:reference':self.parse_list_references, 2075 'list:string':self.parse_list_strings, 2076 }
2077
2078 - def parse(self, rows, fields, colnames, blob_decode=True, 2079 cacheable = False):
2080 db = self.db 2081 virtualtables = [] 2082 new_rows = [] 2083 tmps = [] 2084 for colname in colnames: 2085 if not REGEX_TABLE_DOT_FIELD.match(colname): 2086 tmps.append(None) 2087 else: 2088 (tablename, _the_sep_, fieldname) = colname.partition('.') 2089 table = db[tablename] 2090 field = table[fieldname] 2091 ft = field.type 2092 tmps.append((tablename,fieldname,table,field,ft)) 2093 for (i,row) in enumerate(rows): 2094 new_row = Row() 2095 for (j,colname) in enumerate(colnames): 2096 value = row[j] 2097 tmp = tmps[j] 2098 if tmp: 2099 (tablename,fieldname,table,field,ft) = tmp 2100 if tablename in new_row: 2101 colset = new_row[tablename] 2102 else: 2103 colset = new_row[tablename] = Row() 2104 if tablename not in virtualtables: 2105 virtualtables.append(tablename) 2106 value = self.parse_value(value,ft,blob_decode) 2107 if field.filter_out: 2108 value = field.filter_out(value) 2109 colset[fieldname] = value 2110 2111 # for backward compatibility 2112 if ft=='id' and fieldname!='id' and \ 2113 not 'id' in table.fields: 2114 colset['id'] = value 2115 2116 if ft == 'id' and not cacheable: 2117 # temporary hack to deal with 2118 # GoogleDatastoreAdapter 2119 # references 2120 if isinstance(self, GoogleDatastoreAdapter): 2121 id = value.key().id_or_name() 2122 colset[fieldname] = id 2123 colset.gae_item = value 2124 else: 2125 id = value 2126 colset.update_record = RecordUpdater(colset,table,id) 2127 colset.delete_record = RecordDeleter(table,id) 2128 if table._db._lazy_tables: 2129 colset['__get_lazy_reference__'] = LazyReferenceGetter(table, id) 2130 for rfield in table._referenced_by: 2131 referee_link = db._referee_name and \ 2132 db._referee_name % dict( 2133 table=rfield.tablename,field=rfield.name) 2134 if referee_link and not referee_link in colset: 2135 colset[referee_link] = LazySet(rfield,id) 2136 else: 2137 if not '_extra' in new_row: 2138 new_row['_extra'] = Row() 2139 new_row['_extra'][colname] = \ 2140 self.parse_value(value, 2141 fields[j].type,blob_decode) 2142 new_column_name = \ 2143 REGEX_SELECT_AS_PARSER.search(colname) 2144 if not new_column_name is None: 2145 column_name = new_column_name.groups(0) 2146 setattr(new_row,column_name[0],value) 2147 new_rows.append(new_row) 2148 rowsobj = Rows(db, new_rows, colnames, rawrows=rows) 2149 2150 2151 for tablename in virtualtables: 2152 table = db[tablename] 2153 fields_virtual = [(f,v) for (f,v) in table.iteritems() 2154 if isinstance(v,FieldVirtual)] 2155 fields_lazy = [(f,v) for (f,v) in table.iteritems() 2156 if isinstance(v,FieldMethod)] 2157 if fields_virtual or fields_lazy: 2158 for row in rowsobj.records: 2159 box = row[tablename] 2160 for f,v in fields_virtual: 2161 try: 2162 box[f] = v.f(row) 2163 except AttributeError: 2164 pass # not enough fields to define virtual field 2165 for f,v in fields_lazy: 2166 try: 2167 box[f] = (v.handler or VirtualCommand)(v.f,row) 2168 except AttributeError: 2169 pass # not enough fields to define virtual field 2170 2171 ### old style virtual fields 2172 for item in table.virtualfields: 2173 try: 2174 rowsobj = rowsobj.setvirtualfields(**{tablename:item}) 2175 except (KeyError, AttributeError): 2176 # to avoid breaking virtualfields when partial select 2177 pass 2178 return rowsobj
2179
2180 - def common_filter(self, query, tablenames):
2181 tenant_fieldname = self.db._request_tenant 2182 2183 for tablename in tablenames: 2184 table = self.db[tablename] 2185 2186 # deal with user provided filters 2187 if table._common_filter != None: 2188 query = query & table._common_filter(query) 2189 2190 # deal with multi_tenant filters 2191 if tenant_fieldname in table: 2192 default = table[tenant_fieldname].default 2193 if not default is None: 2194 newquery = table[tenant_fieldname] == default 2195 if query is None: 2196 query = newquery 2197 else: 2198 query = query & newquery 2199 return query
2200
2201 - def CASE(self,query,t,f):
2202 def represent(x): 2203 types = {type(True):'boolean',type(0):'integer',type(1.0):'double'} 2204 if x is None: return 'NULL' 2205 elif isinstance(x,Expression): return str(x) 2206 else: return self.represent(x,types.get(type(x),'string'))
2207 return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \ 2208 (self.expand(query),represent(t),represent(f))) 2209
2210 ################################################################################### 2211 # List of all the available adapters; they all extend BaseAdapter. 2212 ################################################################################### 2213 2214 -class SQLiteAdapter(BaseAdapter):
2215 drivers = ('sqlite2','sqlite3') 2216 2217 can_select_for_update = None # support ourselves with BEGIN TRANSACTION 2218
2219 - def EXTRACT(self,field,what):
2220 return "web2py_extract('%s',%s)" % (what, self.expand(field))
2221 2222 @staticmethod
2223 - def web2py_extract(lookup, s):
2224 table = { 2225 'year': (0, 4), 2226 'month': (5, 7), 2227 'day': (8, 10), 2228 'hour': (11, 13), 2229 'minute': (14, 16), 2230 'second': (17, 19), 2231 } 2232 try: 2233 if lookup != 'epoch': 2234 (i, j) = table[lookup] 2235 return int(s[i:j]) 2236 else: 2237 return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple()) 2238 except: 2239 return None
2240 2241 @staticmethod
2242 - def web2py_regexp(expression, item):
2243 return re.compile(expression).search(item) is not None
2244
2245 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8', 2246 credential_decoder=IDENTITY, driver_args={}, 2247 adapter_args={}, do_connect=True, after_connection=None):
2248 self.db = db 2249 self.dbengine = "sqlite" 2250 self.uri = uri 2251 if do_connect: self.find_driver(adapter_args) 2252 self.pool_size = 0 2253 self.folder = folder 2254 self.db_codec = db_codec 2255 self._after_connection = after_connection 2256 self.find_or_make_work_folder() 2257 path_encoding = sys.getfilesystemencoding() \ 2258 or locale.getdefaultlocale()[1] or 'utf8' 2259 if uri.startswith('sqlite:memory'): 2260 self.dbpath = ':memory:' 2261 else: 2262 self.dbpath = uri.split('://',1)[1] 2263 if self.dbpath[0] != '/': 2264 if PYTHON_VERSION == 2: 2265 self.dbpath = pjoin( 2266 self.folder.decode(path_encoding).encode('utf8'), self.dbpath) 2267 else: 2268 self.dbpath = pjoin(self.folder, self.dbpath) 2269 if not 'check_same_thread' in driver_args: 2270 driver_args['check_same_thread'] = False 2271 if not 'detect_types' in driver_args and do_connect: 2272 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES 2273 def connector(dbpath=self.dbpath, driver_args=driver_args): 2274 return self.driver.Connection(dbpath, **driver_args)
2275 self.connector = connector 2276 if do_connect: self.reconnect()
2277
2278 - def after_connection(self):
2279 self.connection.create_function('web2py_extract', 2, 2280 SQLiteAdapter.web2py_extract) 2281 self.connection.create_function("REGEXP", 2, 2282 SQLiteAdapter.web2py_regexp)
2283
2284 - def _truncate(self, table, mode=''):
2285 tablename = table._tablename 2286 return ['DELETE FROM %s;' % tablename, 2287 "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
2288
2289 - def lastrowid(self, table):
2290 return self.cursor.lastrowid
2291
2292 - def REGEXP(self,first,second):
2293 return '(%s REGEXP %s)' % (self.expand(first), 2294 self.expand(second,'string'))
2295
2296 - def select(self, query, fields, attributes):
2297 """ 2298 Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION. 2299 Note that the entire database, rather than one record, is locked 2300 (it will be locked eventually anyway by the following UPDATE). 2301 """ 2302 if attributes.get('for_update', False) and not 'cache' in attributes: 2303 self.execute('BEGIN IMMEDIATE TRANSACTION;') 2304 return super(SQLiteAdapter, self).select(query, fields, attributes)
2305
2306 -class SpatiaLiteAdapter(SQLiteAdapter):
2307 drivers = ('sqlite3','sqlite2') 2308 2309 types = copy.copy(BaseAdapter.types) 2310 types.update(geometry='GEOMETRY') 2311
2312 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8', 2313 credential_decoder=IDENTITY, driver_args={}, 2314 adapter_args={}, do_connect=True, srid=4326, after_connection=None):
2315 self.db = db 2316 self.dbengine = "spatialite" 2317 self.uri = uri 2318 if do_connect: self.find_driver(adapter_args) 2319 self.pool_size = 0 2320 self.folder = folder 2321 self.db_codec = db_codec 2322 self._after_connection = after_connection 2323 self.find_or_make_work_folder() 2324 self.srid = srid 2325 path_encoding = sys.getfilesystemencoding() \ 2326 or locale.getdefaultlocale()[1] or 'utf8' 2327 if uri.startswith('spatialite:memory'): 2328 self.dbpath = ':memory:' 2329 else: 2330 self.dbpath = uri.split('://',1)[1] 2331 if self.dbpath[0] != '/': 2332 self.dbpath = pjoin( 2333 self.folder.decode(path_encoding).encode('utf8'), self.dbpath) 2334 if not 'check_same_thread' in driver_args: 2335 driver_args['check_same_thread'] = False 2336 if not 'detect_types' in driver_args and do_connect: 2337 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES 2338 def connector(dbpath=self.dbpath, driver_args=driver_args): 2339 return self.driver.Connection(dbpath, **driver_args)
2340 self.connector = connector 2341 if do_connect: self.reconnect()
2342
2343 - def after_connection(self):
2344 self.connection.enable_load_extension(True) 2345 # for Windows, rename libspatialite-2.dll to libspatialite.dll 2346 # Linux uses libspatialite.so 2347 # Mac OS X uses libspatialite.dylib 2348 libspatialite = SPATIALLIBS[platform.system()] 2349 self.execute(r'SELECT load_extension("%s");' % libspatialite) 2350 2351 self.connection.create_function('web2py_extract', 2, 2352 SQLiteAdapter.web2py_extract) 2353 self.connection.create_function("REGEXP", 2, 2354 SQLiteAdapter.web2py_regexp)
2355 2356 # GIS functions 2357
2358 - def ST_ASGEOJSON(self, first, second):
2359 return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first), 2360 second['precision'], second['options'])
2361
2362 - def ST_ASTEXT(self, first):
2363 return 'AsText(%s)' %(self.expand(first))
2364
2365 - def ST_CONTAINS(self, first, second):
2366 return 'Contains(%s,%s)' %(self.expand(first), 2367 self.expand(second, first.type))
2368
2369 - def ST_DISTANCE(self, first, second):
2370 return 'Distance(%s,%s)' %(self.expand(first), 2371 self.expand(second, first.type))
2372
2373 - def ST_EQUALS(self, first, second):
2374 return 'Equals(%s,%s)' %(self.expand(first), 2375 self.expand(second, first.type))
2376
2377 - def ST_INTERSECTS(self, first, second):
2378 return 'Intersects(%s,%s)' %(self.expand(first), 2379 self.expand(second, first.type))
2380
2381 - def ST_OVERLAPS(self, first, second):
2382 return 'Overlaps(%s,%s)' %(self.expand(first), 2383 self.expand(second, first.type))
2384
2385 - def ST_SIMPLIFY(self, first, second):
2386 return 'Simplify(%s,%s)' %(self.expand(first), 2387 self.expand(second, 'double'))
2388
2389 - def ST_TOUCHES(self, first, second):
2390 return 'Touches(%s,%s)' %(self.expand(first), 2391 self.expand(second, first.type))
2392
2393 - def ST_WITHIN(self, first, second):
2394 return 'Within(%s,%s)' %(self.expand(first), 2395 self.expand(second, first.type))
2396
2397 - def represent(self, obj, fieldtype):
2398 field_is_type = fieldtype.startswith 2399 if field_is_type('geo'): 2400 srid = 4326 # Spatialite default srid for geometry 2401 geotype, parms = fieldtype[:-1].split('(') 2402 parms = parms.split(',') 2403 if len(parms) >= 2: 2404 schema, srid = parms[:2] 2405 # if field_is_type('geometry'): 2406 value = "ST_GeomFromText('%s',%s)" %(obj, srid) 2407 # elif field_is_type('geography'): 2408 # value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj) 2409 # else: 2410 # raise SyntaxError, 'Invalid field type %s' %fieldtype 2411 return value 2412 return BaseAdapter.represent(self, obj, fieldtype)
2413
2414 2415 -class JDBCSQLiteAdapter(SQLiteAdapter):
2416 drivers = ('zxJDBC_sqlite',) 2417
2418 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8', 2419 credential_decoder=IDENTITY, driver_args={}, 2420 adapter_args={}, do_connect=True, after_connection=None):
2421 self.db = db 2422 self.dbengine = "sqlite" 2423 self.uri = uri 2424 if do_connect: self.find_driver(adapter_args) 2425 self.pool_size = pool_size 2426 self.folder = folder 2427 self.db_codec = db_codec 2428 self._after_connection = after_connection 2429 self.find_or_make_work_folder() 2430 path_encoding = sys.getfilesystemencoding() \ 2431 or locale.getdefaultlocale()[1] or 'utf8' 2432 if uri.startswith('sqlite:memory'): 2433 self.dbpath = ':memory:' 2434 else: 2435 self.dbpath = uri.split('://',1)[1] 2436 if self.dbpath[0] != '/': 2437 self.dbpath = pjoin( 2438 self.folder.decode(path_encoding).encode('utf8'), self.dbpath) 2439 def connector(dbpath=self.dbpath,driver_args=driver_args): 2440 return self.driver.connect( 2441 self.driver.getConnection('jdbc:sqlite:'+dbpath), 2442 **driver_args)
2443 self.connector = connector 2444 if do_connect: self.reconnect()
2445
2446 - def after_connection(self):
2447 # FIXME http://www.zentus.com/sqlitejdbc/custom_functions.html for UDFs 2448 self.connection.create_function('web2py_extract', 2, 2449 SQLiteAdapter.web2py_extract)
2450
2451 - def execute(self, a):
2452 return self.log_execute(a)
2453
2454 2455 -class MySQLAdapter(BaseAdapter):
2456 drivers = ('MySQLdb','pymysql', 'mysqlconnector') 2457 2458 commit_on_alter_table = True 2459 support_distributed_transaction = True 2460 types = { 2461 'boolean': 'CHAR(1)', 2462 'string': 'VARCHAR(%(length)s)', 2463 'text': 'LONGTEXT', 2464 'json': 'LONGTEXT', 2465 'password': 'VARCHAR(%(length)s)', 2466 'blob': 'LONGBLOB', 2467 'upload': 'VARCHAR(%(length)s)', 2468 'integer': 'INT', 2469 'bigint': 'BIGINT', 2470 'float': 'FLOAT', 2471 'double': 'DOUBLE', 2472 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 2473 'date': 'DATE', 2474 'time': 'TIME', 2475 'datetime': 'DATETIME', 2476 'id': 'INT AUTO_INCREMENT NOT NULL', 2477 'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2478 'list:integer': 'LONGTEXT', 2479 'list:string': 'LONGTEXT', 2480 'list:reference': 'LONGTEXT', 2481 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL', 2482 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2483 } 2484 2485 QUOTE_TEMPLATE = "`%s`" 2486
2487 - def varquote(self,name):
2488 return varquote_aux(name,'`%s`')
2489
2490 - def RANDOM(self):
2491 return 'RAND()'
2492
2493 - def SUBSTRING(self,field,parameters):
2494 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), 2495 parameters[0], parameters[1])
2496
2497 - def EPOCH(self, first):
2498 return "UNIX_TIMESTAMP(%s)" % self.expand(first)
2499
2500 - def CONCAT(self, *items):
2501 return 'CONCAT(%s)' % ','.join(self.expand(x,'string') for x in items)
2502
2503 - def REGEXP(self,first,second):
2504 return '(%s REGEXP %s)' % (self.expand(first), 2505 self.expand(second,'string'))
2506
2507 - def _drop(self,table,mode):
2508 # breaks db integrity but without this mysql does not drop table 2509 return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table, 2510 'SET FOREIGN_KEY_CHECKS=1;']
2511
2512 - def _insert_empty(self, table):
2513 return 'INSERT INTO %s VALUES (DEFAULT);' % table
2514
2515 - def distributed_transaction_begin(self,key):
2516 self.execute('XA START;')
2517
2518 - def prepare(self,key):
2519 self.execute("XA END;") 2520 self.execute("XA PREPARE;")
2521
2522 - def commit_prepared(self,ley):
2523 self.execute("XA COMMIT;")
2524
2525 - def rollback_prepared(self,key):
2526 self.execute("XA ROLLBACK;")
2527 2528 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$') 2529
2530 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 2531 credential_decoder=IDENTITY, driver_args={}, 2532 adapter_args={}, do_connect=True, after_connection=None):
2533 self.db = db 2534 self.dbengine = "mysql" 2535 self.uri = uri 2536 if do_connect: self.find_driver(adapter_args,uri) 2537 self.pool_size = pool_size 2538 self.folder = folder 2539 self.db_codec = db_codec 2540 self._after_connection = after_connection 2541 self.find_or_make_work_folder() 2542 ruri = uri.split('://',1)[1] 2543 m = self.REGEX_URI.match(ruri) 2544 if not m: 2545 raise SyntaxError( 2546 "Invalid URI string in DAL: %s" % self.uri) 2547 user = credential_decoder(m.group('user')) 2548 if not user: 2549 raise SyntaxError('User required') 2550 password = credential_decoder(m.group('password')) 2551 if not password: 2552 password = '' 2553 host = m.group('host') 2554 if not host: 2555 raise SyntaxError('Host name required') 2556 db = m.group('db') 2557 if not db: 2558 raise SyntaxError('Database name required') 2559 port = int(m.group('port') or '3306') 2560 charset = m.group('charset') or 'utf8' 2561 driver_args.update(db=db, 2562 user=credential_decoder(user), 2563 passwd=credential_decoder(password), 2564 host=host, 2565 port=port, 2566 charset=charset) 2567 2568 2569 def connector(driver_args=driver_args): 2570 return self.driver.connect(**driver_args)
2571 self.connector = connector 2572 if do_connect: self.reconnect()
2573
2574 - def after_connection(self):
2575 self.execute('SET FOREIGN_KEY_CHECKS=1;') 2576 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
2577
2578 - def lastrowid(self,table):
2579 self.execute('select last_insert_id();') 2580 return int(self.cursor.fetchone()[0])
2581
2582 2583 -class PostgreSQLAdapter(BaseAdapter):
2584 drivers = ('psycopg2','pg8000') 2585 2586 support_distributed_transaction = True 2587 types = { 2588 'boolean': 'CHAR(1)', 2589 'string': 'VARCHAR(%(length)s)', 2590 'text': 'TEXT', 2591 'json': 'TEXT', 2592 'password': 'VARCHAR(%(length)s)', 2593 'blob': 'BYTEA', 2594 'upload': 'VARCHAR(%(length)s)', 2595 'integer': 'INTEGER', 2596 'bigint': 'BIGINT', 2597 'float': 'FLOAT', 2598 'double': 'FLOAT8', 2599 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 2600 'date': 'DATE', 2601 'time': 'TIME', 2602 'datetime': 'TIMESTAMP', 2603 'id': 'SERIAL PRIMARY KEY', 2604 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2605 'list:integer': 'TEXT', 2606 'list:string': 'TEXT', 2607 'list:reference': 'TEXT', 2608 'geometry': 'GEOMETRY', 2609 'geography': 'GEOGRAPHY', 2610 'big-id': 'BIGSERIAL PRIMARY KEY', 2611 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2612 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2613 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', 2614 2615 } 2616 2617 QUOTE_TEMPLATE = '%s' 2618
2619 - def varquote(self,name):
2620 return varquote_aux(name,'"%s"')
2621
2622 - def adapt(self,obj):
2623 if self.driver_name == 'psycopg2': 2624 return psycopg2_adapt(obj).getquoted() 2625 elif self.driver_name == 'pg8000': 2626 return "'%s'" % str(obj).replace("%","%%").replace("'","''") 2627 else: 2628 return "'%s'" % str(obj).replace("'","''")
2629
2630 - def sequence_name(self,table):
2631 return '%s_id_Seq' % table
2632
2633 - def RANDOM(self):
2634 return 'RANDOM()'
2635
2636 - def ADD(self, first, second):
2637 t = first.type 2638 if t in ('text','string','password', 'json', 'upload','blob'): 2639 return '(%s || %s)' % (self.expand(first), self.expand(second, t)) 2640 else: 2641 return '(%s + %s)' % (self.expand(first), self.expand(second, t))
2642
2643 - def distributed_transaction_begin(self,key):
2644 return
2645
2646 - def prepare(self,key):
2647 self.execute("PREPARE TRANSACTION '%s';" % key)
2648
2649 - def commit_prepared(self,key):
2650 self.execute("COMMIT PREPARED '%s';" % key)
2651
2652 - def rollback_prepared(self,key):
2653 self.execute("ROLLBACK PREPARED '%s';" % key)
2654
2655 - def create_sequence_and_triggers(self, query, table, **args):
2656 # following lines should only be executed if table._sequence_name does not exist 2657 # self.execute('CREATE SEQUENCE %s;' % table._sequence_name) 2658 # self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \ 2659 # % (table._tablename, table._fieldname, table._sequence_name)) 2660 self.execute(query)
2661 2662 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$') 2663
2664 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 2665 credential_decoder=IDENTITY, driver_args={}, 2666 adapter_args={}, do_connect=True, srid=4326, 2667 after_connection=None):
2668 self.db = db 2669 self.dbengine = "postgres" 2670 self.uri = uri 2671 if do_connect: self.find_driver(adapter_args,uri) 2672 self.pool_size = pool_size 2673 self.folder = folder 2674 self.db_codec = db_codec 2675 self._after_connection = after_connection 2676 self.srid = srid 2677 self.find_or_make_work_folder() 2678 ruri = uri.split('://',1)[1] 2679 m = self.REGEX_URI.match(ruri) 2680 if not m: 2681 raise SyntaxError("Invalid URI string in DAL") 2682 user = credential_decoder(m.group('user')) 2683 if not user: 2684 raise SyntaxError('User required') 2685 password = credential_decoder(m.group('password')) 2686 if not password: 2687 password = '' 2688 host = m.group('host') 2689 if not host: 2690 raise SyntaxError('Host name required') 2691 db = m.group('db') 2692 if not db: 2693 raise SyntaxError('Database name required') 2694 port = m.group('port') or '5432' 2695 sslmode = m.group('sslmode') 2696 if sslmode: 2697 msg = ("dbname='%s' user='%s' host='%s' " 2698 "port=%s password='%s' sslmode='%s'") \ 2699 % (db, user, host, port, password, sslmode) 2700 else: 2701 msg = ("dbname='%s' user='%s' host='%s' " 2702 "port=%s password='%s'") \ 2703 % (db, user, host, port, password) 2704 # choose diver according uri 2705 if self.driver: 2706 self.__version__ = "%s %s" % (self.driver.__name__, 2707 self.driver.__version__) 2708 else: 2709 self.__version__ = None 2710 def connector(msg=msg,driver_args=driver_args): 2711 return self.driver.connect(msg,**driver_args)
2712 self.connector = connector 2713 if do_connect: self.reconnect()
2714
2715 - def after_connection(self):
2716 self.connection.set_client_encoding('UTF8') 2717 self.execute("SET standard_conforming_strings=on;") 2718 self.try_json()
2719
2720 - def lastrowid(self,table):
2721 self.execute("select currval('%s')" % table._sequence_name) 2722 return int(self.cursor.fetchone()[0])
2723
2724 - def try_json(self):
2725 # check JSON data type support 2726 # (to be added to after_connection) 2727 if self.driver_name == "pg8000": 2728 supports_json = self.connection.server_version >= "9.2.0" 2729 elif (self.driver_name == "psycopg2") and \ 2730 (self.driver.__version__ >= "2.0.12"): 2731 supports_json = self.connection.server_version >= 90200 2732 elif self.driver_name == "zxJDBC": 2733 supports_json = self.connection.dbversion >= "9.2.0" 2734 else: supports_json = None 2735 if supports_json: 2736 self.types["json"] = "JSON" 2737 self.native_json = True 2738 else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)")
2739
2740 - def LIKE(self,first,second):
2741 args = (self.expand(first), self.expand(second,'string')) 2742 if not first.type in ('string', 'text', 'json'): 2743 return '(%s LIKE %s)' % ( 2744 self.CAST(args[0], 'CHAR(%s)' % first.length), args[1]) 2745 else: 2746 return '(%s LIKE %s)' % args
2747
2748 - def ILIKE(self,first,second):
2749 args = (self.expand(first), self.expand(second,'string')) 2750 if not first.type in ('string', 'text', 'json'): 2751 return '(%s LIKE %s)' % ( 2752 self.CAST(args[0], 'CHAR(%s)' % first.length), args[1]) 2753 else: 2754 return '(%s ILIKE %s)' % args
2755
2756 - def REGEXP(self,first,second):
2757 return '(%s ~ %s)' % (self.expand(first), 2758 self.expand(second,'string'))
2759
2760 - def STARTSWITH(self,first,second):
2761 return '(%s ILIKE %s)' % (self.expand(first), 2762 self.expand(second+'%','string'))
2763
2764 - def ENDSWITH(self,first,second):
2765 return '(%s ILIKE %s)' % (self.expand(first), 2766 self.expand('%'+second,'string'))
2767 2768 # GIS functions 2769
2770 - def ST_ASGEOJSON(self, first, second):
2771 """ 2772 http://postgis.org/docs/ST_AsGeoJSON.html 2773 """ 2774 return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'], 2775 self.expand(first), second['precision'], second['options'])
2776
2777 - def ST_ASTEXT(self, first):
2778 """ 2779 http://postgis.org/docs/ST_AsText.html 2780 """ 2781 return 'ST_AsText(%s)' %(self.expand(first))
2782
2783 - def ST_X(self, first):
2784 """ 2785 http://postgis.org/docs/ST_X.html 2786 """ 2787 return 'ST_X(%s)' %(self.expand(first))
2788
2789 - def ST_Y(self, first):
2790 """ 2791 http://postgis.org/docs/ST_Y.html 2792 """ 2793 return 'ST_Y(%s)' %(self.expand(first))
2794
2795 - def ST_CONTAINS(self, first, second):
2796 """ 2797 http://postgis.org/docs/ST_Contains.html 2798 """ 2799 return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2800
2801 - def ST_DISTANCE(self, first, second):
2802 """ 2803 http://postgis.org/docs/ST_Distance.html 2804 """ 2805 return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2806
2807 - def ST_EQUALS(self, first, second):
2808 """ 2809 http://postgis.org/docs/ST_Equals.html 2810 """ 2811 return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2812
2813 - def ST_INTERSECTS(self, first, second):
2814 """ 2815 http://postgis.org/docs/ST_Intersects.html 2816 """ 2817 return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2818
2819 - def ST_OVERLAPS(self, first, second):
2820 """ 2821 http://postgis.org/docs/ST_Overlaps.html 2822 """ 2823 return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2824
2825 - def ST_SIMPLIFY(self, first, second):
2826 """ 2827 http://postgis.org/docs/ST_Simplify.html 2828 """ 2829 return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
2830
2831 - def ST_TOUCHES(self, first, second):
2832 """ 2833 http://postgis.org/docs/ST_Touches.html 2834 """ 2835 return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2836
2837 - def ST_WITHIN(self, first, second):
2838 """ 2839 http://postgis.org/docs/ST_Within.html 2840 """ 2841 return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2842
2843 - def represent(self, obj, fieldtype):
2844 field_is_type = fieldtype.startswith 2845 if field_is_type('geo'): 2846 srid = 4326 # postGIS default srid for geometry 2847 geotype, parms = fieldtype[:-1].split('(') 2848 parms = parms.split(',') 2849 if len(parms) >= 2: 2850 schema, srid = parms[:2] 2851 if field_is_type('geometry'): 2852 value = "ST_GeomFromText('%s',%s)" %(obj, srid) 2853 elif field_is_type('geography'): 2854 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj) 2855 # else: 2856 # raise SyntaxError('Invalid field type %s' %fieldtype) 2857 return value 2858 return BaseAdapter.represent(self, obj, fieldtype)
2859
2860 -class NewPostgreSQLAdapter(PostgreSQLAdapter):
2861 drivers = ('psycopg2','pg8000') 2862 2863 types = { 2864 'boolean': 'CHAR(1)', 2865 'string': 'VARCHAR(%(length)s)', 2866 'text': 'TEXT', 2867 'json': 'TEXT', 2868 'password': 'VARCHAR(%(length)s)', 2869 'blob': 'BYTEA', 2870 'upload': 'VARCHAR(%(length)s)', 2871 'integer': 'INTEGER', 2872 'bigint': 'BIGINT', 2873 'float': 'FLOAT', 2874 'double': 'FLOAT8', 2875 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 2876 'date': 'DATE', 2877 'time': 'TIME', 2878 'datetime': 'TIMESTAMP', 2879 'id': 'SERIAL PRIMARY KEY', 2880 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2881 'list:integer': 'BIGINT[]', 2882 'list:string': 'TEXT[]', 2883 'list:reference': 'BIGINT[]', 2884 'geometry': 'GEOMETRY', 2885 'geography': 'GEOGRAPHY', 2886 'big-id': 'BIGSERIAL PRIMARY KEY', 2887 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2888 } 2889
2890 - def parse_list_integers(self, value, field_type):
2891 return value
2892
2893 - def parse_list_references(self, value, field_type):
2894 return [self.parse_reference(r, field_type[5:]) for r in value]
2895
2896 - def parse_list_strings(self, value, field_type):
2897 return value
2898
2899 - def represent(self, obj, fieldtype):
2900 field_is_type = fieldtype.startswith 2901 if field_is_type('list:'): 2902 if not obj: 2903 obj = [] 2904 elif not isinstance(obj, (list, tuple)): 2905 obj = [obj] 2906 if field_is_type('list:string'): 2907 obj = map(str,obj) 2908 else: 2909 obj = map(int,obj) 2910 return 'ARRAY[%s]' % ','.join(repr(item) for item in obj) 2911 return BaseAdapter.represent(self, obj, fieldtype)
2912
2913 2914 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
2915 drivers = ('zxJDBC',) 2916 2917 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$') 2918
2919 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 2920 credential_decoder=IDENTITY, driver_args={}, 2921 adapter_args={}, do_connect=True, after_connection=None ):
2922 self.db = db 2923 self.dbengine = "postgres" 2924 self.uri = uri 2925 if do_connect: self.find_driver(adapter_args,uri) 2926 self.pool_size = pool_size 2927 self.folder = folder 2928 self.db_codec = db_codec 2929 self._after_connection = after_connection 2930 self.find_or_make_work_folder() 2931 ruri = uri.split('://',1)[1] 2932 m = self.REGEX_URI.match(ruri) 2933 if not m: 2934 raise SyntaxError("Invalid URI string in DAL") 2935 user = credential_decoder(m.group('user')) 2936 if not user: 2937 raise SyntaxError('User required') 2938 password = credential_decoder(m.group('password')) 2939 if not password: 2940 password = '' 2941 host = m.group('host') 2942 if not host: 2943 raise SyntaxError('Host name required') 2944 db = m.group('db') 2945 if not db: 2946 raise SyntaxError('Database name required') 2947 port = m.group('port') or '5432' 2948 msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password) 2949 def connector(msg=msg,driver_args=driver_args): 2950 return self.driver.connect(*msg,**driver_args)
2951 self.connector = connector 2952 if do_connect: self.reconnect()
2953
2954 - def after_connection(self):
2955 self.connection.set_client_encoding('UTF8') 2956 self.execute('BEGIN;') 2957 self.execute("SET CLIENT_ENCODING TO 'UNICODE';") 2958 self.try_json()
2959
2960 2961 -class OracleAdapter(BaseAdapter):
2962 drivers = ('cx_Oracle',) 2963 2964 commit_on_alter_table = False 2965 types = { 2966 'boolean': 'CHAR(1)', 2967 'string': 'VARCHAR2(%(length)s)', 2968 'text': 'CLOB', 2969 'json': 'CLOB', 2970 'password': 'VARCHAR2(%(length)s)', 2971 'blob': 'CLOB', 2972 'upload': 'VARCHAR2(%(length)s)', 2973 'integer': 'INT', 2974 'bigint': 'NUMBER', 2975 'float': 'FLOAT', 2976 'double': 'BINARY_DOUBLE', 2977 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 2978 'date': 'DATE', 2979 'time': 'CHAR(8)', 2980 'datetime': 'DATE', 2981 'id': 'NUMBER PRIMARY KEY', 2982 'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2983 'list:integer': 'CLOB', 2984 'list:string': 'CLOB', 2985 'list:reference': 'CLOB', 2986 'big-id': 'NUMBER PRIMARY KEY', 2987 'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2988 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 2989 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', 2990 } 2991
2992 - def sequence_name(self,tablename):
2993 return '%s_sequence' % tablename
2994
2995 - def trigger_name(self,tablename):
2996 return '%s_trigger' % tablename
2997
2998 - def LEFT_JOIN(self):
2999 return 'LEFT OUTER JOIN'
3000
3001 - def RANDOM(self):
3002 return 'dbms_random.value'
3003
3004 - def NOT_NULL(self,default,field_type):
3005 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3006
3007 - def _drop(self,table,mode):
3008 sequence_name = table._sequence_name 3009 return ['DROP TABLE %s %s;' % (table, mode), 'DROP SEQUENCE %s;' % sequence_name]
3010
3011 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3012 if limitby: 3013 (lmin, lmax) = limitby 3014 if len(sql_w) > 1: 3015 sql_w_row = sql_w + ' AND w_row > %i' % lmin 3016 else: 3017 sql_w_row = 'WHERE w_row > %i' % lmin 3018 return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o) 3019 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3020
3021 - def constraint_name(self, tablename, fieldname):
3022 constraint_name = BaseAdapter.constraint_name(self, tablename, fieldname) 3023 if len(constraint_name)>30: 3024 constraint_name = '%s_%s__constraint' % (tablename[:10], fieldname[:7]) 3025 return constraint_name
3026
3027 - def represent_exceptions(self, obj, fieldtype):
3028 if fieldtype == 'blob': 3029 obj = base64.b64encode(str(obj)) 3030 return ":CLOB('%s')" % obj 3031 elif fieldtype == 'date': 3032 if isinstance(obj, (datetime.date, datetime.datetime)): 3033 obj = obj.isoformat()[:10] 3034 else: 3035 obj = str(obj) 3036 return "to_date('%s','yyyy-mm-dd')" % obj 3037 elif fieldtype == 'datetime': 3038 if isinstance(obj, datetime.datetime): 3039 obj = obj.isoformat()[:19].replace('T',' ') 3040 elif isinstance(obj, datetime.date): 3041 obj = obj.isoformat()[:10]+' 00:00:00' 3042 else: 3043 obj = str(obj) 3044 return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj 3045 return None
3046
3047 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3048 credential_decoder=IDENTITY, driver_args={}, 3049 adapter_args={}, do_connect=True, after_connection=None):
3050 self.db = db 3051 self.dbengine = "oracle" 3052 self.uri = uri 3053 if do_connect: self.find_driver(adapter_args,uri) 3054 self.pool_size = pool_size 3055 self.folder = folder 3056 self.db_codec = db_codec 3057 self._after_connection = after_connection 3058 self.find_or_make_work_folder() 3059 ruri = uri.split('://',1)[1] 3060 if not 'threaded' in driver_args: 3061 driver_args['threaded']=True 3062 def connector(uri=ruri,driver_args=driver_args): 3063 return self.driver.connect(uri,**driver_args)
3064 self.connector = connector 3065 if do_connect: self.reconnect()
3066
3067 - def after_connection(self):
3068 self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';") 3069 self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
3070 3071 oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))") 3072
3073 - def execute(self, command, args=None):
3074 args = args or [] 3075 i = 1 3076 while True: 3077 m = self.oracle_fix.match(command) 3078 if not m: 3079 break 3080 command = command[:m.start('clob')] + str(i) + command[m.end('clob'):] 3081 args.append(m.group('clob')[6:-2].replace("''", "'")) 3082 i += 1 3083 if command[-1:]==';': 3084 command = command[:-1] 3085 return self.log_execute(command, args)
3086
3087 - def create_sequence_and_triggers(self, query, table, **args):
3088 tablename = table._tablename 3089 id_name = table._id.name 3090 sequence_name = table._sequence_name 3091 trigger_name = table._trigger_name 3092 self.execute(query) 3093 self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name) 3094 self.execute(""" 3095 CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW 3096 DECLARE 3097 curr_val NUMBER; 3098 diff_val NUMBER; 3099 PRAGMA autonomous_transaction; 3100 BEGIN 3101 IF :NEW.%(id)s IS NOT NULL THEN 3102 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val; 3103 diff_val := :NEW.%(id)s - curr_val - 1; 3104 IF diff_val != 0 THEN 3105 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val; 3106 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val; 3107 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1'; 3108 END IF; 3109 END IF; 3110 SELECT %(sequence_name)s.nextval INTO :NEW.%(id)s FROM DUAL; 3111 END; 3112 """ % dict(trigger_name=trigger_name, tablename=tablename, 3113 sequence_name=sequence_name,id=id_name))
3114
3115 - def lastrowid(self,table):
3116 sequence_name = table._sequence_name 3117 self.execute('SELECT %s.currval FROM dual;' % sequence_name) 3118 return long(self.cursor.fetchone()[0])
3119 3120 #def parse_value(self, value, field_type, blob_decode=True): 3121 # if blob_decode and isinstance(value, cx_Oracle.LOB): 3122 # try: 3123 # value = value.read() 3124 # except self.driver.ProgrammingError: 3125 # # After a subsequent fetch the LOB value is not valid anymore 3126 # pass 3127 # return BaseAdapter.parse_value(self, value, field_type, blob_decode) 3128
3129 - def _fetchall(self):
3130 if any(x[1]==cx_Oracle.CLOB for x in self.cursor.description): 3131 return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \ 3132 for c in r]) for r in self.cursor] 3133 else: 3134 return self.cursor.fetchall()
3135
3136 -class MSSQLAdapter(BaseAdapter):
3137 drivers = ('pyodbc',) 3138 T_SEP = 'T' 3139 3140 QUOTE_TEMPLATE = "[%s]" 3141 3142 types = { 3143 'boolean': 'BIT', 3144 'string': 'VARCHAR(%(length)s)', 3145 'text': 'TEXT', 3146 'json': 'TEXT', 3147 'password': 'VARCHAR(%(length)s)', 3148 'blob': 'IMAGE', 3149 'upload': 'VARCHAR(%(length)s)', 3150 'integer': 'INT', 3151 'bigint': 'BIGINT', 3152 'float': 'FLOAT', 3153 'double': 'FLOAT', 3154 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 3155 'date': 'DATETIME', 3156 'time': 'CHAR(8)', 3157 'datetime': 'DATETIME', 3158 'id': 'INT IDENTITY PRIMARY KEY', 3159 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3160 'list:integer': 'TEXT', 3161 'list:string': 'TEXT', 3162 'list:reference': 'TEXT', 3163 'geometry': 'geometry', 3164 'geography': 'geography', 3165 'big-id': 'BIGINT IDENTITY PRIMARY KEY', 3166 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3167 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3168 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', 3169 } 3170
3171 - def concat_add(self,tablename):
3172 return '; ALTER TABLE %s ADD ' % tablename
3173
3174 - def varquote(self,name):
3175 return varquote_aux(name,'[%s]')
3176
3177 - def EXTRACT(self,field,what):
3178 return "DATEPART(%s,%s)" % (what, self.expand(field))
3179
3180 - def LEFT_JOIN(self):
3181 return 'LEFT OUTER JOIN'
3182
3183 - def RANDOM(self):
3184 return 'NEWID()'
3185
3186 - def ALLOW_NULL(self):
3187 return ' NULL'
3188
3189 - def CAST(self, first, second):
3190 return first # apparently no cast necessary in MSSQL
3191
3192 - def SUBSTRING(self,field,parameters):
3193 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
3194
3195 - def PRIMARY_KEY(self,key):
3196 return 'PRIMARY KEY CLUSTERED (%s)' % key
3197
3198 - def AGGREGATE(self, first, what):
3199 if what == 'LENGTH': 3200 what = 'LEN' 3201 return "%s(%s)" % (what, self.expand(first))
3202 3203
3204 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3205 if limitby: 3206 (lmin, lmax) = limitby 3207 sql_s += ' TOP %i' % lmax 3208 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3209 3210 TRUE = 1 3211 FALSE = 0 3212 3213 REGEX_DSN = re.compile('^(?P<dsn>.+)$') 3214 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$') 3215 REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)') 3216
3217 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3218 credential_decoder=IDENTITY, driver_args={}, 3219 adapter_args={}, do_connect=True, srid=4326, 3220 after_connection=None):
3221 self.db = db 3222 self.dbengine = "mssql" 3223 self.uri = uri 3224 if do_connect: self.find_driver(adapter_args,uri) 3225 self.pool_size = pool_size 3226 self.folder = folder 3227 self.db_codec = db_codec 3228 self._after_connection = after_connection 3229 self.srid = srid 3230 self.find_or_make_work_folder() 3231 # ## read: http://bytes.com/groups/python/460325-cx_oracle-utf8 3232 ruri = uri.split('://',1)[1] 3233 if '@' not in ruri: 3234 try: 3235 m = self.REGEX_DSN.match(ruri) 3236 if not m: 3237 raise SyntaxError( 3238 'Parsing uri string(%s) has no result' % self.uri) 3239 dsn = m.group('dsn') 3240 if not dsn: 3241 raise SyntaxError('DSN required') 3242 except SyntaxError: 3243 e = sys.exc_info()[1] 3244 LOGGER.error('NdGpatch error') 3245 raise e 3246 # was cnxn = 'DSN=%s' % dsn 3247 cnxn = dsn 3248 else: 3249 m = self.REGEX_URI.match(ruri) 3250 if not m: 3251 raise SyntaxError( 3252 "Invalid URI string in DAL: %s" % self.uri) 3253 user = credential_decoder(m.group('user')) 3254 if not user: 3255 raise SyntaxError('User required') 3256 password = credential_decoder(m.group('password')) 3257 if not password: 3258 password = '' 3259 host = m.group('host') 3260 if not host: 3261 raise SyntaxError('Host name required') 3262 db = m.group('db') 3263 if not db: 3264 raise SyntaxError('Database name required') 3265 port = m.group('port') or '1433' 3266 # Parse the optional url name-value arg pairs after the '?' 3267 # (in the form of arg1=value1&arg2=value2&...) 3268 # Default values (drivers like FreeTDS insist on uppercase parameter keys) 3269 argsdict = { 'DRIVER':'{SQL Server}' } 3270 urlargs = m.group('urlargs') or '' 3271 for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs): 3272 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue') 3273 urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()]) 3274 cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \ 3275 % (host, port, db, user, password, urlargs) 3276 def connector(cnxn=cnxn,driver_args=driver_args): 3277 return self.driver.connect(cnxn,**driver_args)
3278 self.connector = connector 3279 if do_connect: self.reconnect()
3280
3281 - def lastrowid(self,table):
3282 #self.execute('SELECT @@IDENTITY;') 3283 self.execute('SELECT SCOPE_IDENTITY();') 3284 return long(self.cursor.fetchone()[0])
3285
3286 - def rowslice(self,rows,minimum=0,maximum=None):
3287 if maximum is None: 3288 return rows[minimum:] 3289 return rows[minimum:maximum]
3290
3291 - def EPOCH(self, first):
3292 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3293
3294 - def CONCAT(self, *items):
3295 return '(%s)' % ' + '.join(self.expand(x,'string') for x in items)
3296 3297 # GIS Spatial Extensions 3298 3299 # No STAsGeoJSON in MSSQL 3300
3301 - def ST_ASTEXT(self, first):
3302 return '%s.STAsText()' %(self.expand(first))
3303
3304 - def ST_CONTAINS(self, first, second):
3305 return '%s.STContains(%s)=1' %(self.expand(first), self.expand(second, first.type))
3306
3307 - def ST_DISTANCE(self, first, second):
3308 return '%s.STDistance(%s)' %(self.expand(first), self.expand(second, first.type))
3309
3310 - def ST_EQUALS(self, first, second):
3311 return '%s.STEquals(%s)=1' %(self.expand(first), self.expand(second, first.type))
3312
3313 - def ST_INTERSECTS(self, first, second):
3314 return '%s.STIntersects(%s)=1' %(self.expand(first), self.expand(second, first.type))
3315
3316 - def ST_OVERLAPS(self, first, second):
3317 return '%s.STOverlaps(%s)=1' %(self.expand(first), self.expand(second, first.type))
3318 3319 # no STSimplify in MSSQL 3320
3321 - def ST_TOUCHES(self, first, second):
3322 return '%s.STTouches(%s)=1' %(self.expand(first), self.expand(second, first.type))
3323
3324 - def ST_WITHIN(self, first, second):
3325 return '%s.STWithin(%s)=1' %(self.expand(first), self.expand(second, first.type))
3326
3327 - def represent(self, obj, fieldtype):
3328 field_is_type = fieldtype.startswith 3329 if field_is_type('geometry'): 3330 srid = 0 # MS SQL default srid for geometry 3331 geotype, parms = fieldtype[:-1].split('(') 3332 if parms: 3333 srid = parms 3334 return "geometry::STGeomFromText('%s',%s)" %(obj, srid) 3335 elif fieldtype == 'geography': 3336 srid = 4326 # MS SQL default srid for geography 3337 geotype, parms = fieldtype[:-1].split('(') 3338 if parms: 3339 srid = parms 3340 return "geography::STGeomFromText('%s',%s)" %(obj, srid) 3341 # else: 3342 # raise SyntaxError('Invalid field type %s' %fieldtype) 3343 return "geometry::STGeomFromText('%s',%s)" %(obj, srid) 3344 return BaseAdapter.represent(self, obj, fieldtype)
3345
3346 3347 -class MSSQL3Adapter(MSSQLAdapter):
3348 """ experimental support for pagination in MSSQL"""
3349 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3350 if limitby: 3351 (lmin, lmax) = limitby 3352 if lmin == 0: 3353 sql_s += ' TOP %i' % lmax 3354 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o) 3355 lmin += 1 3356 sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:] 3357 sql_g_inner = sql_o[:sql_o.find('ORDER BY ')] 3358 sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))] 3359 sql_f_inner = [f for f in sql_f.split(',')] 3360 sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)] 3361 sql_f_iproxy = ', '.join(sql_f_iproxy) 3362 sql_f_oproxy = ', '.join(sql_f_outer) 3363 return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax) 3364 return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
3365 - def rowslice(self,rows,minimum=0,maximum=None):
3366 return rows
3367
3368 3369 -class MSSQL2Adapter(MSSQLAdapter):
3370 drivers = ('pyodbc',) 3371 3372 types = { 3373 'boolean': 'CHAR(1)', 3374 'string': 'NVARCHAR(%(length)s)', 3375 'text': 'NTEXT', 3376 'json': 'NTEXT', 3377 'password': 'NVARCHAR(%(length)s)', 3378 'blob': 'IMAGE', 3379 'upload': 'NVARCHAR(%(length)s)', 3380 'integer': 'INT', 3381 'bigint': 'BIGINT', 3382 'float': 'FLOAT', 3383 'double': 'FLOAT', 3384 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 3385 'date': 'DATETIME', 3386 'time': 'CHAR(8)', 3387 'datetime': 'DATETIME', 3388 'id': 'INT IDENTITY PRIMARY KEY', 3389 'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3390 'list:integer': 'NTEXT', 3391 'list:string': 'NTEXT', 3392 'list:reference': 'NTEXT', 3393 'big-id': 'BIGINT IDENTITY PRIMARY KEY', 3394 'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3395 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3396 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', 3397 } 3398
3399 - def represent(self, obj, fieldtype):
3400 value = BaseAdapter.represent(self, obj, fieldtype) 3401 if fieldtype in ('string','text', 'json') and value[:1]=="'": 3402 value = 'N'+value 3403 return value
3404
3405 - def execute(self,a):
3406 return self.log_execute(a.decode('utf8'))
3407
3408 -class VerticaAdapter(MSSQLAdapter):
3409 drivers = ('pyodbc',) 3410 T_SEP = ' ' 3411 3412 types = { 3413 'boolean': 'BOOLEAN', 3414 'string': 'VARCHAR(%(length)s)', 3415 'text': 'BYTEA', 3416 'json': 'VARCHAR(%(length)s)', 3417 'password': 'VARCHAR(%(length)s)', 3418 'blob': 'BYTEA', 3419 'upload': 'VARCHAR(%(length)s)', 3420 'integer': 'INT', 3421 'bigint': 'BIGINT', 3422 'float': 'FLOAT', 3423 'double': 'DOUBLE PRECISION', 3424 'decimal': 'DECIMAL(%(precision)s,%(scale)s)', 3425 'date': 'DATE', 3426 'time': 'TIME', 3427 'datetime': 'DATETIME', 3428 'id': 'IDENTITY', 3429 'reference': 'INT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3430 'list:integer': 'BYTEA', 3431 'list:string': 'BYTEA', 3432 'list:reference': 'BYTEA', 3433 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3434 } 3435 3436
3437 - def EXTRACT(self, first, what):
3438 return "DATE_PART('%s', TIMESTAMP %s)" % (what, self.expand(first))
3439
3440 - def _truncate(self, table, mode=''):
3441 tablename = table._tablename 3442 return ['TRUNCATE %s %s;' % (tablename, mode or '')]
3443
3444 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3445 if limitby: 3446 (lmin, lmax) = limitby 3447 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin) 3448 return 'SELECT %s %s FROM %s%s%s;' % \ 3449 (sql_s, sql_f, sql_t, sql_w, sql_o)
3450
3451 - def lastrowid(self,table):
3452 self.execute('SELECT LAST_INSERT_ID();') 3453 return long(self.cursor.fetchone()[0])
3454
3455 - def execute(self, a):
3456 return self.log_execute(a)
3457
3458 -class SybaseAdapter(MSSQLAdapter):
3459 drivers = ('Sybase',) 3460 3461 types = { 3462 'boolean': 'BIT', 3463 'string': 'CHAR VARYING(%(length)s)', 3464 'text': 'TEXT', 3465 'json': 'TEXT', 3466 'password': 'CHAR VARYING(%(length)s)', 3467 'blob': 'IMAGE', 3468 'upload': 'CHAR VARYING(%(length)s)', 3469 'integer': 'INT', 3470 'bigint': 'BIGINT', 3471 'float': 'FLOAT', 3472 'double': 'FLOAT', 3473 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 3474 'date': 'DATETIME', 3475 'time': 'CHAR(8)', 3476 'datetime': 'DATETIME', 3477 'id': 'INT IDENTITY PRIMARY KEY', 3478 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3479 'list:integer': 'TEXT', 3480 'list:string': 'TEXT', 3481 'list:reference': 'TEXT', 3482 'geometry': 'geometry', 3483 'geography': 'geography', 3484 'big-id': 'BIGINT IDENTITY PRIMARY KEY', 3485 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3486 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3487 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', 3488 } 3489 3490
3491 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3492 credential_decoder=IDENTITY, driver_args={}, 3493 adapter_args={}, do_connect=True, srid=4326, 3494 after_connection=None):
3495 self.db = db 3496 self.dbengine = "sybase" 3497 self.uri = uri 3498 if do_connect: self.find_driver(adapter_args,uri) 3499 self.pool_size = pool_size 3500 self.folder = folder 3501 self.db_codec = db_codec 3502 self._after_connection = after_connection 3503 self.srid = srid 3504 self.find_or_make_work_folder() 3505 # ## read: http://bytes.com/groups/python/460325-cx_oracle-utf8 3506 ruri = uri.split('://',1)[1] 3507 if '@' not in ruri: 3508 try: 3509 m = self.REGEX_DSN.match(ruri) 3510 if not m: 3511 raise SyntaxError( 3512 'Parsing uri string(%s) has no result' % self.uri) 3513 dsn = m.group('dsn') 3514 if not dsn: 3515 raise SyntaxError('DSN required') 3516 except SyntaxError: 3517 e = sys.exc_info()[1] 3518 LOGGER.error('NdGpatch error') 3519 raise e 3520 else: 3521 m = self.REGEX_URI.match(uri) 3522 if not m: 3523 raise SyntaxError( 3524 "Invalid URI string in DAL: %s" % self.uri) 3525 user = credential_decoder(m.group('user')) 3526 if not user: 3527 raise SyntaxError('User required') 3528 password = credential_decoder(m.group('password')) 3529 if not password: 3530 password = '' 3531 host = m.group('host') 3532 if not host: 3533 raise SyntaxError('Host name required') 3534 db = m.group('db') 3535 if not db: 3536 raise SyntaxError('Database name required') 3537 port = m.group('port') or '1433' 3538 3539 dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db) 3540 3541 driver_args.update(user = credential_decoder(user), 3542 password = credential_decoder(password)) 3543 3544 def connector(dsn=dsn,driver_args=driver_args): 3545 return self.driver.connect(dsn,**driver_args)
3546 self.connector = connector 3547 if do_connect: self.reconnect()
3548
3549 3550 -class FireBirdAdapter(BaseAdapter):
3551 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc') 3552 3553 commit_on_alter_table = False 3554 support_distributed_transaction = True 3555 types = { 3556 'boolean': 'CHAR(1)', 3557 'string': 'VARCHAR(%(length)s)', 3558 'text': 'BLOB SUB_TYPE 1', 3559 'json': 'BLOB SUB_TYPE 1', 3560 'password': 'VARCHAR(%(length)s)', 3561 'blob': 'BLOB SUB_TYPE 0', 3562 'upload': 'VARCHAR(%(length)s)', 3563 'integer': 'INTEGER', 3564 'bigint': 'BIGINT', 3565 'float': 'FLOAT', 3566 'double': 'DOUBLE PRECISION', 3567 'decimal': 'DECIMAL(%(precision)s,%(scale)s)', 3568 'date': 'DATE', 3569 'time': 'TIME', 3570 'datetime': 'TIMESTAMP', 3571 'id': 'INTEGER PRIMARY KEY', 3572 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3573 'list:integer': 'BLOB SUB_TYPE 1', 3574 'list:string': 'BLOB SUB_TYPE 1', 3575 'list:reference': 'BLOB SUB_TYPE 1', 3576 'big-id': 'BIGINT PRIMARY KEY', 3577 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3578 } 3579
3580 - def sequence_name(self,tablename):
3581 return 'genid_%s' % tablename
3582
3583 - def trigger_name(self,tablename):
3584 return 'trg_id_%s' % tablename
3585
3586 - def RANDOM(self):
3587 return 'RAND()'
3588
3589 - def EPOCH(self, first):
3590 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3591
3592 - def NOT_NULL(self,default,field_type):
3593 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3594
3595 - def SUBSTRING(self,field,parameters):
3596 return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
3597
3598 - def LENGTH(self, first):
3599 return "CHAR_LENGTH(%s)" % self.expand(first)
3600
3601 - def CONTAINS(self,first,second,case_sensitive=False):
3602 if first.type.startswith('list:'): 3603 second = Expression(None,self.CONCAT('|',Expression( 3604 None,self.REPLACE(second,('|','||'))),'|')) 3605 return '(%s CONTAINING %s)' % (self.expand(first), 3606 self.expand(second, 'string'))
3607
3608 - def _drop(self,table,mode):
3609 sequence_name = table._sequence_name 3610 return ['DROP TABLE %s %s;' % (table, mode), 'DROP GENERATOR %s;' % sequence_name]
3611
3612 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3613 if limitby: 3614 (lmin, lmax) = limitby 3615 sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s) 3616 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3617
3618 - def _truncate(self,table,mode = ''):
3619 return ['DELETE FROM %s;' % table._tablename, 3620 'SET GENERATOR %s TO 0;' % table._sequence_name]
3621 3622 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$') 3623
3624 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3625 credential_decoder=IDENTITY, driver_args={}, 3626 adapter_args={}, do_connect=True, after_connection=None):
3627 self.db = db 3628 self.dbengine = "firebird" 3629 self.uri = uri 3630 if do_connect: self.find_driver(adapter_args,uri) 3631 self.pool_size = pool_size 3632 self.folder = folder 3633 self.db_codec = db_codec 3634 self._after_connection = after_connection 3635 self.find_or_make_work_folder() 3636 ruri = uri.split('://',1)[1] 3637 m = self.REGEX_URI.match(ruri) 3638 if not m: 3639 raise SyntaxError("Invalid URI string in DAL: %s" % self.uri) 3640 user = credential_decoder(m.group('user')) 3641 if not user: 3642 raise SyntaxError('User required') 3643 password = credential_decoder(m.group('password')) 3644 if not password: 3645 password = '' 3646 host = m.group('host') 3647 if not host: 3648 raise SyntaxError('Host name required') 3649 port = int(m.group('port') or 3050) 3650 db = m.group('db') 3651 if not db: 3652 raise SyntaxError('Database name required') 3653 charset = m.group('charset') or 'UTF8' 3654 driver_args.update(dsn='%s/%s:%s' % (host,port,db), 3655 user = credential_decoder(user), 3656 password = credential_decoder(password), 3657 charset = charset) 3658 3659 def connector(driver_args=driver_args): 3660 return self.driver.connect(**driver_args)
3661 self.connector = connector 3662 if do_connect: self.reconnect()
3663
3664 - def create_sequence_and_triggers(self, query, table, **args):
3665 tablename = table._tablename 3666 sequence_name = table._sequence_name 3667 trigger_name = table._trigger_name 3668 self.execute(query) 3669 self.execute('create generator %s;' % sequence_name) 3670 self.execute('set generator %s to 0;' % sequence_name) 3671 self.execute('create trigger %s for %s active before insert position 0 as\nbegin\nif(new.id is null) then\nbegin\nnew.id = gen_id(%s, 1);\nend\nend;' % (trigger_name, tablename, sequence_name))
3672
3673 - def lastrowid(self,table):
3674 sequence_name = table._sequence_name 3675 self.execute('SELECT gen_id(%s, 0) FROM rdb$database' % sequence_name) 3676 return long(self.cursor.fetchone()[0])
3677
3678 3679 -class FireBirdEmbeddedAdapter(FireBirdAdapter):
3680 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc') 3681 3682 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$') 3683
3684 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3685 credential_decoder=IDENTITY, driver_args={}, 3686 adapter_args={}, do_connect=True, after_connection=None):
3687 self.db = db 3688 self.dbengine = "firebird" 3689 self.uri = uri 3690 if do_connect: self.find_driver(adapter_args,uri) 3691 self.pool_size = pool_size 3692 self.folder = folder 3693 self.db_codec = db_codec 3694 self._after_connection = after_connection 3695 self.find_or_make_work_folder() 3696 ruri = uri.split('://',1)[1] 3697 m = self.REGEX_URI.match(ruri) 3698 if not m: 3699 raise SyntaxError( 3700 "Invalid URI string in DAL: %s" % self.uri) 3701 user = credential_decoder(m.group('user')) 3702 if not user: 3703 raise SyntaxError('User required') 3704 password = credential_decoder(m.group('password')) 3705 if not password: 3706 password = '' 3707 pathdb = m.group('path') 3708 if not pathdb: 3709 raise SyntaxError('Path required') 3710 charset = m.group('charset') 3711 if not charset: 3712 charset = 'UTF8' 3713 host = '' 3714 driver_args.update(host=host, 3715 database=pathdb, 3716 user=credential_decoder(user), 3717 password=credential_decoder(password), 3718 charset=charset) 3719 3720 def connector(driver_args=driver_args): 3721 return self.driver.connect(**driver_args)
3722 self.connector = connector 3723 if do_connect: self.reconnect()
3724
3725 -class InformixAdapter(BaseAdapter):
3726 drivers = ('informixdb',) 3727 3728 types = { 3729 'boolean': 'CHAR(1)', 3730 'string': 'VARCHAR(%(length)s)', 3731 'text': 'BLOB SUB_TYPE 1', 3732 'json': 'BLOB SUB_TYPE 1', 3733 'password': 'VARCHAR(%(length)s)', 3734 'blob': 'BLOB SUB_TYPE 0', 3735 'upload': 'VARCHAR(%(length)s)', 3736 'integer': 'INTEGER', 3737 'bigint': 'BIGINT', 3738 'float': 'FLOAT', 3739 'double': 'DOUBLE PRECISION', 3740 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 3741 'date': 'DATE', 3742 'time': 'CHAR(8)', 3743 'datetime': 'DATETIME', 3744 'id': 'SERIAL', 3745 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3746 'list:integer': 'BLOB SUB_TYPE 1', 3747 'list:string': 'BLOB SUB_TYPE 1', 3748 'list:reference': 'BLOB SUB_TYPE 1', 3749 'big-id': 'BIGSERIAL', 3750 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3751 'reference FK': 'REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s CONSTRAINT FK_%(table_name)s_%(field_name)s', 3752 'reference TFK': 'FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s CONSTRAINT TFK_%(table_name)s_%(field_name)s', 3753 } 3754
3755 - def RANDOM(self):
3756 return 'Random()'
3757
3758 - def NOT_NULL(self,default,field_type):
3759 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3760
3761 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3762 if limitby: 3763 (lmin, lmax) = limitby 3764 fetch_amt = lmax - lmin 3765 dbms_version = int(self.connection.dbms_version.split('.')[0]) 3766 if lmin and (dbms_version >= 10): 3767 # Requires Informix 10.0+ 3768 sql_s += ' SKIP %d' % (lmin, ) 3769 if fetch_amt and (dbms_version >= 9): 3770 # Requires Informix 9.0+ 3771 sql_s += ' FIRST %d' % (fetch_amt, ) 3772 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3773
3774 - def represent_exceptions(self, obj, fieldtype):
3775 if fieldtype == 'date': 3776 if isinstance(obj, (datetime.date, datetime.datetime)): 3777 obj = obj.isoformat()[:10] 3778 else: 3779 obj = str(obj) 3780 return "to_date('%s','%%Y-%%m-%%d')" % obj 3781 elif fieldtype == 'datetime': 3782 if isinstance(obj, datetime.datetime): 3783 obj = obj.isoformat()[:19].replace('T',' ') 3784 elif isinstance(obj, datetime.date): 3785 obj = obj.isoformat()[:10]+' 00:00:00' 3786 else: 3787 obj = str(obj) 3788 return "to_date('%s','%%Y-%%m-%%d %%H:%%M:%%S')" % obj 3789 return None
3790 3791 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$') 3792
3793 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3794 credential_decoder=IDENTITY, driver_args={}, 3795 adapter_args={}, do_connect=True, after_connection=None):
3796 self.db = db 3797 self.dbengine = "informix" 3798 self.uri = uri 3799 if do_connect: self.find_driver(adapter_args,uri) 3800 self.pool_size = pool_size 3801 self.folder = folder 3802 self.db_codec = db_codec 3803 self._after_connection = after_connection 3804 self.find_or_make_work_folder() 3805 ruri = uri.split('://',1)[1] 3806 m = self.REGEX_URI.match(ruri) 3807 if not m: 3808 raise SyntaxError( 3809 "Invalid URI string in DAL: %s" % self.uri) 3810 user = credential_decoder(m.group('user')) 3811 if not user: 3812 raise SyntaxError('User required') 3813 password = credential_decoder(m.group('password')) 3814 if not password: 3815 password = '' 3816 host = m.group('host') 3817 if not host: 3818 raise SyntaxError('Host name required') 3819 db = m.group('db') 3820 if not db: 3821 raise SyntaxError('Database name required') 3822 user = credential_decoder(user) 3823 password = credential_decoder(password) 3824 dsn = '%s@%s' % (db,host) 3825 driver_args.update(user=user,password=password,autocommit=True) 3826 def connector(dsn=dsn,driver_args=driver_args): 3827 return self.driver.connect(dsn,**driver_args)
3828 self.connector = connector 3829 if do_connect: self.reconnect()
3830
3831 - def execute(self,command):
3832 if command[-1:]==';': 3833 command = command[:-1] 3834 return self.log_execute(command)
3835
3836 - def lastrowid(self,table):
3837 return self.cursor.sqlerrd[1]
3838
3839 -class InformixSEAdapter(InformixAdapter):
3840 """ work in progress """ 3841
3842 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3843 return 'SELECT %s %s FROM %s%s%s;' % \ 3844 (sql_s, sql_f, sql_t, sql_w, sql_o)
3845
3846 - def rowslice(self,rows,minimum=0,maximum=None):
3847 if maximum is None: 3848 return rows[minimum:] 3849 return rows[minimum:maximum]
3850
3851 -class DB2Adapter(BaseAdapter):
3852 drivers = ('pyodbc',) 3853 3854 types = { 3855 'boolean': 'CHAR(1)', 3856 'string': 'VARCHAR(%(length)s)', 3857 'text': 'CLOB', 3858 'json': 'CLOB', 3859 'password': 'VARCHAR(%(length)s)', 3860 'blob': 'BLOB', 3861 'upload': 'VARCHAR(%(length)s)', 3862 'integer': 'INT', 3863 'bigint': 'BIGINT', 3864 'float': 'REAL', 3865 'double': 'DOUBLE', 3866 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 3867 'date': 'DATE', 3868 'time': 'TIME', 3869 'datetime': 'TIMESTAMP', 3870 'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL', 3871 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3872 'list:integer': 'CLOB', 3873 'list:string': 'CLOB', 3874 'list:reference': 'CLOB', 3875 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL', 3876 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3877 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 3878 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', 3879 } 3880
3881 - def LEFT_JOIN(self):
3882 return 'LEFT OUTER JOIN'
3883
3884 - def RANDOM(self):
3885 return 'RAND()'
3886
3887 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3888 if limitby: 3889 (lmin, lmax) = limitby 3890 sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax 3891 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3892
3893 - def represent_exceptions(self, obj, fieldtype):
3894 if fieldtype == 'blob': 3895 obj = base64.b64encode(str(obj)) 3896 return "BLOB('%s')" % obj 3897 elif fieldtype == 'datetime': 3898 if isinstance(obj, datetime.datetime): 3899 obj = obj.isoformat()[:19].replace('T','-').replace(':','.') 3900 elif isinstance(obj, datetime.date): 3901 obj = obj.isoformat()[:10]+'-00.00.00' 3902 return "'%s'" % obj 3903 return None
3904
3905 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3906 credential_decoder=IDENTITY, driver_args={}, 3907 adapter_args={}, do_connect=True, after_connection=None):
3908 self.db = db 3909 self.dbengine = "db2" 3910 self.uri = uri 3911 if do_connect: self.find_driver(adapter_args,uri) 3912 self.pool_size = pool_size 3913 self.folder = folder 3914 self.db_codec = db_codec 3915 self._after_connection = after_connection 3916 self.find_or_make_work_folder() 3917 ruri = uri.split('://', 1)[1] 3918 def connector(cnxn=ruri,driver_args=driver_args): 3919 return self.driver.connect(cnxn,**driver_args)
3920 self.connector = connector 3921 if do_connect: self.reconnect()
3922
3923 - def execute(self,command):
3924 if command[-1:]==';': 3925 command = command[:-1] 3926 return self.log_execute(command)
3927
3928 - def lastrowid(self,table):
3929 self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table) 3930 return long(self.cursor.fetchone()[0])
3931
3932 - def rowslice(self,rows,minimum=0,maximum=None):
3933 if maximum is None: 3934 return rows[minimum:] 3935 return rows[minimum:maximum]
3936
3937 3938 -class TeradataAdapter(BaseAdapter):
3939 drivers = ('pyodbc',) 3940 3941 types = { 3942 'boolean': 'CHAR(1)', 3943 'string': 'VARCHAR(%(length)s)', 3944 'text': 'CLOB', 3945 'json': 'CLOB', 3946 'password': 'VARCHAR(%(length)s)', 3947 'blob': 'BLOB', 3948 'upload': 'VARCHAR(%(length)s)', 3949 'integer': 'INT', 3950 'bigint': 'BIGINT', 3951 'float': 'REAL', 3952 'double': 'DOUBLE', 3953 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 3954 'date': 'DATE', 3955 'time': 'TIME', 3956 'datetime': 'TIMESTAMP', 3957 # Modified Constraint syntax for Teradata. 3958 # Teradata does not support ON DELETE. 3959 'id': 'INT GENERATED ALWAYS AS IDENTITY', # Teradata Specific 3960 'reference': 'INT', 3961 'list:integer': 'CLOB', 3962 'list:string': 'CLOB', 3963 'list:reference': 'CLOB', 3964 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY', # Teradata Specific 3965 'big-reference': 'BIGINT', 3966 'reference FK': ' REFERENCES %(foreign_key)s', 3967 'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)', 3968 } 3969
3970 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 3971 credential_decoder=IDENTITY, driver_args={}, 3972 adapter_args={}, do_connect=True, after_connection=None):
3973 self.db = db 3974 self.dbengine = "teradata" 3975 self.uri = uri 3976 if do_connect: self.find_driver(adapter_args,uri) 3977 self.pool_size = pool_size 3978 self.folder = folder 3979 self.db_codec = db_codec 3980 self._after_connection = after_connection 3981 self.find_or_make_work_folder() 3982 ruri = uri.split('://', 1)[1] 3983 def connector(cnxn=ruri,driver_args=driver_args): 3984 return self.driver.connect(cnxn,**driver_args)
3985 self.connector = connector 3986 if do_connect: self.reconnect()
3987
3988 - def LEFT_JOIN(self):
3989 return 'LEFT OUTER JOIN'
3990 3991 # Similar to MSSQL, Teradata can't specify a range (for Pageby)
3992 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3993 if limitby: 3994 (lmin, lmax) = limitby 3995 sql_s += ' TOP %i' % lmax 3996 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3997
3998 - def _truncate(self, table, mode=''):
3999 tablename = table._tablename 4000 return ['DELETE FROM %s ALL;' % (tablename)]
4001 4002 INGRES_SEQNAME='ii***lineitemsequence' # NOTE invalid database object name
4003 # (ANSI-SQL wants this form of name 4004 # to be a delimited identifier) 4005 4006 -class IngresAdapter(BaseAdapter):
4007 drivers = ('pyodbc',) 4008 4009 types = { 4010 'boolean': 'CHAR(1)', 4011 'string': 'VARCHAR(%(length)s)', 4012 'text': 'CLOB', 4013 'json': 'CLOB', 4014 'password': 'VARCHAR(%(length)s)', ## Not sure what this contains utf8 or nvarchar. Or even bytes? 4015 'blob': 'BLOB', 4016 'upload': 'VARCHAR(%(length)s)', ## FIXME utf8 or nvarchar... or blob? what is this type? 4017 'integer': 'INTEGER4', # or int8... 4018 'bigint': 'BIGINT', 4019 'float': 'FLOAT', 4020 'double': 'FLOAT8', 4021 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 4022 'date': 'ANSIDATE', 4023 'time': 'TIME WITHOUT TIME ZONE', 4024 'datetime': 'TIMESTAMP WITHOUT TIME ZONE', 4025 'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME, 4026 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4027 'list:integer': 'CLOB', 4028 'list:string': 'CLOB', 4029 'list:reference': 'CLOB', 4030 'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME, 4031 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4032 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4033 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO 4034 } 4035
4036 - def LEFT_JOIN(self):
4037 return 'LEFT OUTER JOIN'
4038
4039 - def RANDOM(self):
4040 return 'RANDOM()'
4041
4042 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4043 if limitby: 4044 (lmin, lmax) = limitby 4045 fetch_amt = lmax - lmin 4046 if fetch_amt: 4047 sql_s += ' FIRST %d ' % (fetch_amt, ) 4048 if lmin: 4049 # Requires Ingres 9.2+ 4050 sql_o += ' OFFSET %d' % (lmin, ) 4051 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4052
4053 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 4054 credential_decoder=IDENTITY, driver_args={}, 4055 adapter_args={}, do_connect=True, after_connection=None):
4056 self.db = db 4057 self.dbengine = "ingres" 4058 self._driver = pyodbc 4059 self.uri = uri 4060 if do_connect: self.find_driver(adapter_args,uri) 4061 self.pool_size = pool_size 4062 self.folder = folder 4063 self.db_codec = db_codec 4064 self._after_connection = after_connection 4065 self.find_or_make_work_folder() 4066 connstr = uri.split(':', 1)[1] 4067 # Simple URI processing 4068 connstr = connstr.lstrip() 4069 while connstr.startswith('/'): 4070 connstr = connstr[1:] 4071 if '=' in connstr: 4072 # Assume we have a regular ODBC connection string and just use it 4073 ruri = connstr 4074 else: 4075 # Assume only (local) dbname is passed in with OS auth 4076 database_name = connstr 4077 default_driver_name = 'Ingres' 4078 vnode = '(local)' 4079 servertype = 'ingres' 4080 ruri = 'Driver={%s};Server=%s;Database=%s' % (default_driver_name, vnode, database_name) 4081 def connector(cnxn=ruri,driver_args=driver_args): 4082 return self.driver.connect(cnxn,**driver_args)
4083 4084 self.connector = connector 4085 4086 # TODO if version is >= 10, set types['id'] to Identity column, see http://community.actian.com/wiki/Using_Ingres_Identity_Columns 4087 if do_connect: self.reconnect()
4088
4089 - def create_sequence_and_triggers(self, query, table, **args):
4090 # post create table auto inc code (if needed) 4091 # modify table to btree for performance.... 4092 # Older Ingres releases could use rule/trigger like Oracle above. 4093 if hasattr(table,'_primarykey'): 4094 modify_tbl_sql = 'modify %s to btree unique on %s' % \ 4095 (table._tablename, 4096 ', '.join(["'%s'" % x for x in table.primarykey])) 4097 self.execute(modify_tbl_sql) 4098 else: 4099 tmp_seqname='%s_iisq' % table._tablename 4100 query=query.replace(INGRES_SEQNAME, tmp_seqname) 4101 self.execute('create sequence %s' % tmp_seqname) 4102 self.execute(query) 4103 self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
4104 4105
4106 - def lastrowid(self,table):
4107 tmp_seqname='%s_iisq' % table 4108 self.execute('select current value for %s' % tmp_seqname) 4109 return long(self.cursor.fetchone()[0]) # don't really need int type cast here...
4110
4111 4112 -class IngresUnicodeAdapter(IngresAdapter):
4113 4114 drivers = ('pyodbc',) 4115 4116 types = { 4117 'boolean': 'CHAR(1)', 4118 'string': 'NVARCHAR(%(length)s)', 4119 'text': 'NCLOB', 4120 'json': 'NCLOB', 4121 'password': 'NVARCHAR(%(length)s)', ## Not sure what this contains utf8 or nvarchar. Or even bytes? 4122 'blob': 'BLOB', 4123 'upload': 'VARCHAR(%(length)s)', ## FIXME utf8 or nvarchar... or blob? what is this type? 4124 'integer': 'INTEGER4', # or int8... 4125 'bigint': 'BIGINT', 4126 'float': 'FLOAT', 4127 'double': 'FLOAT8', 4128 'decimal': 'NUMERIC(%(precision)s,%(scale)s)', 4129 'date': 'ANSIDATE', 4130 'time': 'TIME WITHOUT TIME ZONE', 4131 'datetime': 'TIMESTAMP WITHOUT TIME ZONE', 4132 'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME, 4133 'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4134 'list:integer': 'NCLOB', 4135 'list:string': 'NCLOB', 4136 'list:reference': 'NCLOB', 4137 'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME, 4138 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4139 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4140 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO 4141 }
4142
4143 -class SAPDBAdapter(BaseAdapter):
4144 drivers = ('sapdb',) 4145 4146 support_distributed_transaction = False 4147 types = { 4148 'boolean': 'CHAR(1)', 4149 'string': 'VARCHAR(%(length)s)', 4150 'text': 'LONG', 4151 'json': 'LONG', 4152 'password': 'VARCHAR(%(length)s)', 4153 'blob': 'LONG', 4154 'upload': 'VARCHAR(%(length)s)', 4155 'integer': 'INT', 4156 'bigint': 'BIGINT', 4157 'float': 'FLOAT', 4158 'double': 'DOUBLE PRECISION', 4159 'decimal': 'FIXED(%(precision)s,%(scale)s)', 4160 'date': 'DATE', 4161 'time': 'TIME', 4162 'datetime': 'TIMESTAMP', 4163 'id': 'INT PRIMARY KEY', 4164 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4165 'list:integer': 'LONG', 4166 'list:string': 'LONG', 4167 'list:reference': 'LONG', 4168 'big-id': 'BIGINT PRIMARY KEY', 4169 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', 4170 } 4171
4172 - def sequence_name(self,table):
4173 return '%s_id_Seq' % table
4174
4175 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4176 if limitby: 4177 (lmin, lmax) = limitby 4178 if len(sql_w) > 1: 4179 sql_w_row = sql_w + ' AND w_row > %i' % lmin 4180 else: 4181 sql_w_row = 'WHERE w_row > %i' % lmin 4182 return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o) 4183 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4184
4185 - def create_sequence_and_triggers(self, query, table, **args):
4186 # following lines should only be executed if table._sequence_name does not exist 4187 self.execute('CREATE SEQUENCE %s;' % table._sequence_name) 4188 self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \ 4189 % (table._tablename, table._id.name, table._sequence_name)) 4190 self.execute(query)
4191 4192 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$') 4193 4194
4195 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 4196 credential_decoder=IDENTITY, driver_args={}, 4197 adapter_args={}, do_connect=True, after_connection=None):
4198 self.db = db 4199 self.dbengine = "sapdb" 4200 self.uri = uri 4201 if do_connect: self.find_driver(adapter_args,uri) 4202 self.pool_size = pool_size 4203 self.folder = folder 4204 self.db_codec = db_codec 4205 self._after_connection = after_connection 4206 self.find_or_make_work_folder() 4207 ruri = uri.split('://',1)[1] 4208 m = self.REGEX_URI.match(ruri) 4209 if not m: 4210 raise SyntaxError("Invalid URI string in DAL") 4211 user = credential_decoder(m.group('user')) 4212 if not user: 4213 raise SyntaxError('User required') 4214 password = credential_decoder(m.group('password')) 4215 if not password: 4216 password = '' 4217 host = m.group('host') 4218 if not host: 4219 raise SyntaxError('Host name required') 4220 db = m.group('db') 4221 if not db: 4222 raise SyntaxError('Database name required') 4223 def connector(user=user, password=password, database=db, 4224 host=host, driver_args=driver_args): 4225 return self.driver.Connection(user, password, database, 4226 host, **driver_args)
4227 self.connector = connector 4228 if do_connect: self.reconnect()
4229
4230 - def lastrowid(self,table):
4231 self.execute("select %s.NEXTVAL from dual" % table._sequence_name) 4232 return long(self.cursor.fetchone()[0])
4233
4234 -class CubridAdapter(MySQLAdapter):
4235 drivers = ('cubriddb',) 4236 4237 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$') 4238
4239 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8', 4240 credential_decoder=IDENTITY, driver_args={}, 4241 adapter_args={}, do_connect=True, after_connection=None):
4242 self.db = db 4243 self.dbengine = "cubrid" 4244 self.uri = uri 4245 if do_connect: self.find_driver(adapter_args,uri) 4246 self.pool_size = pool_size 4247 self.folder = folder 4248 self.db_codec = db_codec 4249 self._after_connection = after_connection 4250 self.find_or_make_work_folder() 4251 ruri = uri.split('://',1)[1] 4252 m = self.REGEX_URI.match(ruri) 4253 if not m: 4254 raise SyntaxError( 4255 "Invalid URI string in DAL: %s" % self.uri) 4256 user = credential_decoder(m.group('user')) 4257 if not user: 4258 raise SyntaxError('User required') 4259 password = credential_decoder(m.group('password')) 4260 if not password: 4261 password = '' 4262 host = m.group('host') 4263 if not host: 4264 raise SyntaxError('Host name required') 4265 db = m.group('db') 4266 if not db: 4267 raise SyntaxError('Database name required') 4268 port = int(m.group('port') or '30000') 4269 charset = m.group('charset') or 'utf8' 4270 user = credential_decoder(user) 4271 passwd = credential_decoder(password) 4272 def connector(host=host,port=port,db=db, 4273 user=user,passwd=password,driver_args=driver_args): 4274 return self.driver.connect(host,port,db,user,passwd,**driver_args)
4275 self.connector = connector 4276 if do_connect: self.reconnect()
4277
4278 - def after_connection(self):
4279 self.execute('SET FOREIGN_KEY_CHECKS=1;') 4280 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4281
4282 4283 ######## GAE MySQL ########## 4284 4285 -class DatabaseStoredFile:
4286 4287 web2py_filesystem = False 4288
4289 - def escape(self,obj):
4290 return self.db._adapter.escape(obj)
4291
4292 - def __init__(self,db,filename,mode):
4293 if not db._adapter.dbengine in ('mysql', 'postgres', 'sqlite'): 4294 raise RuntimeError("only MySQL/Postgres/SQLite can store metadata .table files in database for now") 4295 self.db = db 4296 self.filename = filename 4297 self.mode = mode 4298 if not self.web2py_filesystem: 4299 if db._adapter.dbengine == 'mysql': 4300 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;" 4301 elif db._adapter.dbengine in ('postgres', 'sqlite'): 4302 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));" 4303 self.db.executesql(sql) 4304 DatabaseStoredFile.web2py_filesystem = True 4305 self.p=0 4306 self.data = '' 4307 if mode in ('r','rw','a'): 4308 query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \ 4309 % filename 4310 rows = self.db.executesql(query) 4311 if rows: 4312 self.data = rows[0][0] 4313 elif exists(filename): 4314 datafile = open(filename, 'r') 4315 try: 4316 self.data = datafile.read() 4317 finally: 4318 datafile.close() 4319 elif mode in ('r','rw'): 4320 raise RuntimeError("File %s does not exist" % filename)
4321
4322 - def read(self, bytes):
4323 data = self.data[self.p:self.p+bytes] 4324 self.p += len(data) 4325 return data
4326
4327 - def readline(self):
4328 i = self.data.find('\n',self.p)+1 4329 if i>0: 4330 data, self.p = self.data[self.p:i], i 4331 else: 4332 data, self.p = self.data[self.p:], len(self.data) 4333 return data
4334
4335 - def write(self,data):
4336 self.data += data
4337
4338 - def close_connection(self):
4339 if self.db is not None: 4340 self.db.executesql( 4341 "DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename) 4342 query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\ 4343 % (self.filename, self.data.replace("'","''")) 4344 self.db.executesql(query) 4345 self.db.commit() 4346 self.db = None
4347
4348 - def close(self):
4349 self.close_connection()
4350 4351 @staticmethod
4352 - def exists(db, filename):
4353 if exists(filename): 4354 return True 4355 query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename 4356 try: 4357 if db.executesql(query): 4358 return True 4359 except Exception, e: 4360 if not (db._adapter.isOperationalError(e) or 4361 db._adapter.isProgrammingError(e)): 4362 raise 4363 # no web2py_filesystem found? 4364 tb = traceback.format_exc() 4365 LOGGER.error("Could not retrieve %s\n%s" % (filename, tb)) 4366 return False
4367
4368 4369 -class UseDatabaseStoredFile:
4370
4371 - def file_exists(self, filename):
4372 return DatabaseStoredFile.exists(self.db,filename)
4373
4374 - def file_open(self, filename, mode='rb', lock=True):
4375 return DatabaseStoredFile(self.db,filename,mode)
4376
4377 - def file_close(self, fileobj):
4378 fileobj.close_connection()
4379
4380 - def file_delete(self,filename):
4381 query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename 4382 self.db.executesql(query) 4383 self.db.commit()
4384
4385 -class GoogleSQLAdapter(UseDatabaseStoredFile,MySQLAdapter):
4386 uploads_in_blob = True 4387 4388 REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$') 4389
4390 - def __init__(self, db, uri='google:sql://realm:domain/database', 4391 pool_size=0, folder=None, db_codec='UTF-8', 4392 credential_decoder=IDENTITY, driver_args={}, 4393 adapter_args={}, do_connect=True, after_connection=None):
4394 4395 self.db = db 4396 self.dbengine = "mysql" 4397 self.uri = uri 4398 self.pool_size = pool_size 4399 self.db_codec = db_codec 4400 self._after_connection = after_connection 4401 if do_connect: self.find_driver(adapter_args, uri) 4402 self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split( 4403 os.sep+'applications'+os.sep,1)[1]) 4404 ruri = uri.split("://")[1] 4405 m = self.REGEX_URI.match(ruri) 4406 if not m: 4407 raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri) 4408 instance = credential_decoder(m.group('instance')) 4409 self.dbstring = db = credential_decoder(m.group('db')) 4410 driver_args['instance'] = instance 4411 if not 'charset' in driver_args: 4412 driver_args['charset'] = 'utf8' 4413 self.createdb = createdb = adapter_args.get('createdb',True) 4414 if not createdb: 4415 driver_args['database'] = db 4416 def connector(driver_args=driver_args): 4417 return rdbms.connect(**driver_args)
4418 self.connector = connector 4419 if do_connect: self.reconnect()
4420
4421 - def after_connection(self):
4422 if self.createdb: 4423 # self.execute('DROP DATABASE %s' % self.dbstring) 4424 self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring) 4425 self.execute('USE %s' % self.dbstring) 4426 self.execute("SET FOREIGN_KEY_CHECKS=1;") 4427 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4428
4429 - def execute(self, command, *a, **b):
4430 return self.log_execute(command.decode('utf8'), *a, **b)
4431
4432 - def find_driver(self,adapter_args,uri=None):
4433 self.adapter_args = adapter_args 4434 self.driver = "google"
4435
4436 -class NoSQLAdapter(BaseAdapter):
4437 can_select_for_update = False 4438 4439 @staticmethod
4440 - def to_unicode(obj):
4441 if isinstance(obj, str): 4442 return obj.decode('utf8') 4443 elif not isinstance(obj, unicode): 4444 return unicode(obj) 4445 return obj
4446
4447 - def id_query(self, table):
4448 return table._id > 0
4449
4450 - def represent(self, obj, fieldtype):
4451 field_is_type = fieldtype.startswith 4452 if isinstance(obj, CALLABLETYPES): 4453 obj = obj() 4454 if isinstance(fieldtype, SQLCustomType): 4455 return fieldtype.encoder(obj) 4456 if isinstance(obj, (Expression, Field)): 4457 raise SyntaxError("non supported on GAE") 4458 if self.dbengine == 'google:datastore': 4459 if isinstance(fieldtype, gae.Property): 4460 return obj 4461 is_string = isinstance(fieldtype,str) 4462 is_list = is_string and field_is_type('list:') 4463 if is_list: 4464 if not obj: 4465 obj = [] 4466 if not isinstance(obj, (list, tuple)): 4467 obj = [obj] 4468 if obj == '' and not \ 4469 (is_string and fieldtype[:2] in ['st','te', 'pa','up']): 4470 return None 4471 if not obj is None: 4472 if isinstance(obj, list) and not is_list: 4473 obj = [self.represent(o, fieldtype) for o in obj] 4474 elif fieldtype in ('integer','bigint','id'): 4475 obj = long(obj) 4476 elif fieldtype == 'double': 4477 obj = float(obj) 4478 elif is_string and field_is_type('reference'): 4479 if isinstance(obj, (Row, Reference)): 4480 obj = obj['id'] 4481 obj = long(obj) 4482 elif fieldtype == 'boolean': 4483 if obj and not str(obj)[0].upper() in '0F': 4484 obj = True 4485 else: 4486 obj = False 4487 elif fieldtype == 'date': 4488 if not isinstance(obj, datetime.date): 4489 (y, m, d) = map(int,str(obj).strip().split('-')) 4490 obj = datetime.date(y, m, d) 4491 elif isinstance(obj,datetime.datetime): 4492 (y, m, d) = (obj.year, obj.month, obj.day) 4493 obj = datetime.date(y, m, d) 4494 elif fieldtype == 'time': 4495 if not isinstance(obj, datetime.time): 4496 time_items = map(int,str(obj).strip().split(':')[:3]) 4497 if len(time_items) == 3: 4498 (h, mi, s) = time_items 4499 else: 4500 (h, mi, s) = time_items + [0] 4501 obj = datetime.time(h, mi, s) 4502 elif fieldtype == 'datetime': 4503 if not isinstance(obj, datetime.datetime): 4504 (y, m, d) = map(int,str(obj)[:10].strip().split('-')) 4505 time_items = map(int,str(obj)[11:].strip().split(':')[:3]) 4506 while len(time_items)<3: 4507 time_items.append(0) 4508 (h, mi, s) = time_items 4509 obj = datetime.datetime(y, m, d, h, mi, s) 4510 elif fieldtype == 'blob': 4511 pass 4512 elif fieldtype == 'json': 4513 if isinstance(obj, basestring): 4514 obj = self.to_unicode(obj) 4515 if have_serializers: 4516 obj = serializers.loads_json(obj) 4517 elif simplejson: 4518 obj = simplejson.loads(obj) 4519 else: 4520 raise RuntimeError("missing simplejson") 4521 elif is_string and field_is_type('list:string'): 4522 return map(self.to_unicode,obj) 4523 elif is_list: 4524 return map(int,obj) 4525 else: 4526 obj = self.to_unicode(obj) 4527 return obj
4528
4529 - def _insert(self,table,fields):
4530 return 'insert %s in %s' % (fields, table)
4531
4532 - def _count(self,query,distinct=None):
4533 return 'count %s' % repr(query)
4534
4535 - def _select(self,query,fields,attributes):
4536 return 'select %s where %s' % (repr(fields), repr(query))
4537
4538 - def _delete(self,tablename, query):
4539 return 'delete %s where %s' % (repr(tablename),repr(query))
4540
4541 - def _update(self,tablename,query,fields):
4542 return 'update %s (%s) where %s' % (repr(tablename), 4543 repr(fields),repr(query))
4544
4545 - def commit(self):
4546 """ 4547 remember: no transactions on many NoSQL 4548 """ 4549 pass
4550
4551 - def rollback(self):
4552 """ 4553 remember: no transactions on many NoSQL 4554 """ 4555 pass
4556
4557 - def close_connection(self):
4558 """ 4559 remember: no transactions on many NoSQL 4560 """ 4561 pass
4562 4563 4564 # these functions should never be called!
4565 - def OR(self,first,second): raise SyntaxError("Not supported")
4566 - def AND(self,first,second): raise SyntaxError("Not supported")
4567 - def AS(self,first,second): raise SyntaxError("Not supported")
4568 - def ON(self,first,second): raise SyntaxError("Not supported")
4569 - def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported")
4570 - def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported")
4571 - def ADD(self,first,second): raise SyntaxError("Not supported")
4572 - def SUB(self,first,second): raise SyntaxError("Not supported")
4573 - def MUL(self,first,second): raise SyntaxError("Not supported")
4574 - def DIV(self,first,second): raise SyntaxError("Not supported")
4575 - def LOWER(self,first): raise SyntaxError("Not supported")
4576 - def UPPER(self,first): raise SyntaxError("Not supported")
4577 - def EXTRACT(self,first,what): raise SyntaxError("Not supported")
4578 - def LENGTH(self, first): raise SyntaxError("Not supported")
4579 - def AGGREGATE(self,first,what): raise SyntaxError("Not supported")
4580 - def LEFT_JOIN(self): raise SyntaxError("Not supported")
4581 - def RANDOM(self): raise SyntaxError("Not supported")
4582 - def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported")
4583 - def PRIMARY_KEY(self,key): raise SyntaxError("Not supported")
4584 - def ILIKE(self,first,second): raise SyntaxError("Not supported")
4585 - def drop(self,table,mode): raise SyntaxError("Not supported")
4586 - def alias(self,table,alias): raise SyntaxError("Not supported")
4587 - def migrate_table(self,*a,**b): raise SyntaxError("Not supported")
4588 - def distributed_transaction_begin(self,key): raise SyntaxError("Not supported")
4589 - def prepare(self,key): raise SyntaxError("Not supported")
4590 - def commit_prepared(self,key): raise SyntaxError("Not supported")
4591 - def rollback_prepared(self,key): raise SyntaxError("Not supported")
4592 - def concat_add(self,table): raise SyntaxError("Not supported")
4593 - def constraint_name(self, table, fieldname): raise SyntaxError("Not supported")
4594 - def create_sequence_and_triggers(self, query, table, **args): pass
4595 - def log_execute(self,*a,**b): raise SyntaxError("Not supported")
4596 - def execute(self,*a,**b): raise SyntaxError("Not supported")
4597 - def represent_exceptions(self, obj, fieldtype): raise SyntaxError("Not supported")
4598 - def lastrowid(self,table): raise SyntaxError("Not supported")
4599 - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported")
4600
4601 4602 -class GAEF(object):
4603 - def __init__(self,name,op,value,apply):
4604 self.name=name=='id' and '__key__' or name 4605 self.op=op 4606 self.value=value 4607 self.apply=apply
4608 - def __repr__(self):
4609 return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
4610
4611 -class GoogleDatastoreAdapter(NoSQLAdapter):
4612 uploads_in_blob = True 4613 types = {} 4614
4615 - def file_exists(self, filename): pass
4616 - def file_open(self, filename, mode='rb', lock=True): pass
4617 - def file_close(self, fileobj): pass
4618 4619 REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)') 4620
4621 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', 4622 credential_decoder=IDENTITY, driver_args={}, 4623 adapter_args={}, do_connect=True, after_connection=None):
4624 self.types.update({ 4625 'boolean': gae.BooleanProperty, 4626 'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)), 4627 'text': gae.TextProperty, 4628 'json': gae.TextProperty, 4629 'password': gae.StringProperty, 4630 'blob': gae.BlobProperty, 4631 'upload': gae.StringProperty, 4632 'integer': gae.IntegerProperty, 4633 'bigint': gae.IntegerProperty, 4634 'float': gae.FloatProperty, 4635 'double': gae.FloatProperty, 4636 'decimal': GAEDecimalProperty, 4637 'date': gae.DateProperty, 4638 'time': gae.TimeProperty, 4639 'datetime': gae.DateTimeProperty, 4640 'id': None, 4641 'reference': gae.IntegerProperty, 4642 'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)), 4643 'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)), 4644 'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)), 4645 }) 4646 self.db = db 4647 self.uri = uri 4648 self.dbengine = 'google:datastore' 4649 self.folder = folder 4650 db['_lastsql'] = '' 4651 self.db_codec = 'UTF-8' 4652 self._after_connection = after_connection 4653 self.pool_size = 0 4654 match = self.REGEX_NAMESPACE.match(uri) 4655 if match: 4656 namespace_manager.set_namespace(match.group('namespace'))
4657
4658 - def parse_id(self, value, field_type):
4659 return value
4660
4661 - def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
4662 myfields = {} 4663 for field in table: 4664 if isinstance(polymodel,Table) and field.name in polymodel.fields(): 4665 continue 4666 attr = {} 4667 if isinstance(field.custom_qualifier, dict): 4668 #this is custom properties to add to the GAE field declartion 4669 attr = field.custom_qualifier 4670 field_type = field.type 4671 if isinstance(field_type, SQLCustomType): 4672 ftype = self.types[field_type.native or field_type.type](**attr) 4673 elif isinstance(field_type, gae.Property): 4674 ftype = field_type 4675 elif field_type.startswith('id'): 4676 continue 4677 elif field_type.startswith('decimal'): 4678 precision, scale = field_type[7:].strip('()').split(',') 4679 precision = int(precision) 4680 scale = int(scale) 4681 ftype = GAEDecimalProperty(precision, scale, **attr) 4682 elif field_type.startswith('reference'): 4683 if field.notnull: 4684 attr = dict(required=True) 4685 referenced = field_type[10:].strip() 4686 ftype = self.types[field_type[:9]](referenced, **attr) 4687 elif field_type.startswith('list:reference'): 4688 if field.notnull: 4689 attr['required'] = True 4690 referenced = field_type[15:].strip() 4691 ftype = self.types[field_type[:14]](**attr) 4692 elif field_type.startswith('list:'): 4693 ftype = self.types[field_type](**attr) 4694 elif not field_type in self.types\ 4695 or not self.types[field_type]: 4696 raise SyntaxError('Field: unknown field type: %s' % field_type) 4697 else: 4698 ftype = self.types[field_type](**attr) 4699 myfields[field.name] = ftype 4700 if not polymodel: 4701 table._tableobj = classobj(table._tablename, (gae.Model, ), myfields) 4702 elif polymodel==True: 4703 table._tableobj = classobj(table._tablename, (PolyModel, ), myfields) 4704 elif isinstance(polymodel,Table): 4705 table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields) 4706 else: 4707 raise SyntaxError("polymodel must be None, True, a table or a tablename") 4708 return None
4709
4710 - def expand(self,expression,field_type=None):
4711 if isinstance(expression,Field): 4712 if expression.type in ('text', 'blob', 'json'): 4713 raise SyntaxError('AppEngine does not index by: %s' % expression.type) 4714 return expression.name 4715 elif isinstance(expression, (Expression, Query)): 4716 if not expression.second is None: 4717 return expression.op(expression.first, expression.second) 4718 elif not expression.first is None: 4719 return expression.op(expression.first) 4720 else: 4721 return expression.op() 4722 elif field_type: 4723 return self.represent(expression,field_type) 4724 elif isinstance(expression,(list,tuple)): 4725 return ','.join([self.represent(item,field_type) for item in expression]) 4726 else: 4727 return str(expression)
4728 4729 ### TODO from gql.py Expression
4730 - def AND(self,first,second):
4731 a = self.expand(first) 4732 b = self.expand(second) 4733 if b[0].name=='__key__' and a[0].name!='__key__': 4734 return b+a 4735 return a+b
4736
4737 - def EQ(self,first,second=None):
4738 if isinstance(second, Key): 4739 return [GAEF(first.name,'=',second,lambda a,b:a==b)] 4740 return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
4741
4742 - def NE(self,first,second=None):
4743 if first.type != 'id': 4744 return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)] 4745 else: 4746 if not second is None: 4747 second = Key.from_path(first._tablename, long(second)) 4748 return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
4749
4750 - def LT(self,first,second=None):
4751 if first.type != 'id': 4752 return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)] 4753 else: 4754 second = Key.from_path(first._tablename, long(second)) 4755 return [GAEF(first.name,'<',second,lambda a,b:a<b)]
4756
4757 - def LE(self,first,second=None):
4758 if first.type != 'id': 4759 return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)] 4760 else: 4761 second = Key.from_path(first._tablename, long(second)) 4762 return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
4763
4764 - def GT(self,first,second=None):
4765 if first.type != 'id' or second==0 or second == '0': 4766 return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)] 4767 else: 4768 second = Key.from_path(first._tablename, long(second)) 4769 return [GAEF(first.name,'>',second,lambda a,b:a>b)]
4770
4771 - def GE(self,first,second=None):
4772 if first.type != 'id': 4773 return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)] 4774 else: 4775 second = Key.from_path(first._tablename, long(second)) 4776 return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
4777
4778 - def INVERT(self,first):
4779 return '-%s' % first.name
4780
4781 - def COMMA(self,first,second):
4782 return '%s, %s' % (self.expand(first),self.expand(second))
4783
4784 - def BELONGS(self,first,second=None):
4785 if not isinstance(second,(list, tuple)): 4786 raise SyntaxError("Not supported") 4787 if first.type != 'id': 4788 return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)] 4789 else: 4790 second = [Key.from_path(first._tablename, int(i)) for i in second] 4791 return [GAEF(first.name,'in',second,lambda a,b:a in b)]
4792
4793 - def CONTAINS(self,first,second,case_sensitive=False):
4794 # silently ignoring: GAE can only do case sensitive matches! 4795 if not first.type.startswith('list:'): 4796 raise SyntaxError("Not supported") 4797 return [GAEF(first.name,'=',self.expand(second,first.type[5:]),lambda a,b:b in a)]
4798
4799 - def NOT(self,first):
4800 nops = { self.EQ: self.NE, 4801 self.NE: self.EQ, 4802 self.LT: self.GE, 4803 self.GT: self.LE, 4804 self.LE: self.GT, 4805 self.GE: self.LT} 4806 if not isinstance(first,Query): 4807 raise SyntaxError("Not suported") 4808 nop = nops.get(first.op,None) 4809 if not nop: 4810 raise SyntaxError("Not suported %s" % first.op.__name__) 4811 first.op = nop 4812 return self.expand(first)
4813
4814 - def truncate(self,table,mode):
4815 self.db(self.db._adapter.id_query(table)).delete()
4816
4817 - def select_raw(self,query,fields=None,attributes=None):
4818 db = self.db 4819 fields = fields or [] 4820 attributes = attributes or {} 4821 args_get = attributes.get 4822 new_fields = [] 4823 for item in fields: 4824 if isinstance(item,SQLALL): 4825 new_fields += item._table 4826 else: 4827 new_fields.append(item) 4828 fields = new_fields 4829 if query: 4830 tablename = self.get_table(query) 4831 elif fields: 4832 tablename = fields[0].tablename 4833 query = db._adapter.id_query(fields[0].table) 4834 else: 4835 raise SyntaxError("Unable to determine a tablename") 4836 4837 if query: 4838 if use_common_filters(query): 4839 query = self.common_filter(query,[tablename]) 4840 4841 #tableobj is a GAE Model class (or subclass) 4842 tableobj = db[tablename]._tableobj 4843 filters = self.expand(query) 4844 4845 projection = None 4846 if len(db[tablename].fields) == len(fields): 4847 #getting all fields, not a projection query 4848 projection = None 4849 elif args_get('projection') == True: 4850 projection = [] 4851 for f in fields: 4852 if f.type in ['text', 'blob', 'json']: 4853 raise SyntaxError( 4854 "text and blob field types not allowed in projection queries") 4855 else: 4856 projection.append(f.name) 4857 elif args_get('filterfields') == True: 4858 projection = [] 4859 for f in fields: 4860 projection.append(f.name) 4861 4862 # real projection's can't include 'id'. 4863 # it will be added to the result later 4864 query_projection = [ 4865 p for p in projection if \ 4866 p != db[tablename]._id.name] if projection and \ 4867 args_get('projection') == True\ 4868 else None 4869 4870 cursor = None 4871 if isinstance(args_get('reusecursor'), str): 4872 cursor = args_get('reusecursor') 4873 items = gae.Query(tableobj, projection=query_projection, 4874 cursor=cursor) 4875 4876 for filter in filters: 4877 if args_get('projection') == True and \ 4878 filter.name in query_projection and \ 4879 filter.op in ['=', '<=', '>=']: 4880 raise SyntaxError( 4881 "projection fields cannot have equality filters") 4882 if filter.name=='__key__' and filter.op=='>' and filter.value==0: 4883 continue 4884 elif filter.name=='__key__' and filter.op=='=': 4885 if filter.value==0: 4886 items = [] 4887 elif isinstance(filter.value, Key): 4888 # key qeuries return a class instance, 4889 # can't use projection 4890 # extra values will be ignored in post-processing later 4891 item = tableobj.get(filter.value) 4892 items = (item and [item]) or [] 4893 else: 4894 # key qeuries return a class instance, 4895 # can't use projection 4896 # extra values will be ignored in post-processing later 4897 item = tableobj.get_by_id(filter.value) 4898 items = (item and [item]) or [] 4899 elif isinstance(items,list): # i.e. there is a single record! 4900 items = [i for i in items if filter.apply( 4901 getattr(item,filter.name),filter.value)] 4902 else: 4903 if filter.name=='__key__' and filter.op != 'in': 4904 items.order('__key__') 4905 items = items.filter('%s %s' % (filter.name,filter.op), 4906 filter.value) 4907 if not isinstance(items,list): 4908 if args_get('left', None): 4909 raise SyntaxError('Set: no left join in appengine') 4910 if args_get('groupby', None): 4911 raise SyntaxError('Set: no groupby in appengine') 4912 orderby = args_get('orderby', False) 4913 if orderby: 4914 ### THIS REALLY NEEDS IMPROVEMENT !!! 4915 if isinstance(orderby, (list, tuple)): 4916 orderby = xorify(orderby) 4917 if isinstance(orderby,Expression): 4918 orderby = self.expand(orderby) 4919 orders = orderby.split(', ') 4920 for order in orders: 4921 order={'-id':'-__key__','id':'__key__'}.get(order,order) 4922 items = items.order(order) 4923 if args_get('limitby', None): 4924 (lmin, lmax) = attributes['limitby'] 4925 (limit, offset) = (lmax - lmin, lmin) 4926 rows = items.fetch(limit,offset=offset) 4927 #cursor is only useful if there was a limit and we didn't return 4928 # all results 4929 if args_get('reusecursor'): 4930 db['_lastcursor'] = items.cursor() 4931 items = rows 4932 return (items, tablename, projection or db[tablename].fields)
4933
4934 - def select(self,query,fields,attributes):
4935 """ 4936 This is the GAE version of select. some notes to consider: 4937 - db['_lastsql'] is not set because there is not SQL statement string 4938 for a GAE query 4939 - 'nativeRef' is a magical fieldname used for self references on GAE 4940 - optional attribute 'projection' when set to True will trigger 4941 use of the GAE projection queries. note that there are rules for 4942 what is accepted imposed by GAE: each field must be indexed, 4943 projection queries cannot contain blob or text fields, and you 4944 cannot use == and also select that same field. see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection 4945 - optional attribute 'filterfields' when set to True web2py will only 4946 parse the explicitly listed fields into the Rows object, even though 4947 all fields are returned in the query. This can be used to reduce 4948 memory usage in cases where true projection queries are not 4949 usable. 4950 - optional attribute 'reusecursor' allows use of cursor with queries 4951 that have the limitby attribute. Set the attribute to True for the 4952 first query, set it to the value of db['_lastcursor'] to continue 4953 a previous query. The user must save the cursor value between 4954 requests, and the filters must be identical. It is up to the user 4955 to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors 4956 """ 4957 4958 (items, tablename, fields) = self.select_raw(query,fields,attributes) 4959 # self.db['_lastsql'] = self._select(query,fields,attributes) 4960 rows = [[(t==self.db[tablename]._id.name and item) or \ 4961 (t=='nativeRef' and item) or getattr(item, t) \ 4962 for t in fields] for item in items] 4963 colnames = ['%s.%s' % (tablename, t) for t in fields] 4964 processor = attributes.get('processor',self.parse) 4965 return processor(rows,fields,colnames,False)
4966
4967 - def count(self,query,distinct=None,limit=None):
4968 if distinct: 4969 raise RuntimeError("COUNT DISTINCT not supported") 4970 (items, tablename, fields) = self.select_raw(query) 4971 # self.db['_lastsql'] = self._count(query) 4972 try: 4973 return len(items) 4974 except TypeError: 4975 return items.count(limit=limit)
4976
4977 - def delete(self,tablename, query):
4978 """ 4979 This function was changed on 2010-05-04 because according to 4980 http://code.google.com/p/googleappengine/issues/detail?id=3119 4981 GAE no longer supports deleting more than 1000 records. 4982 """ 4983 # self.db['_lastsql'] = self._delete(tablename,query) 4984 (items, tablename, fields) = self.select_raw(query) 4985 # items can be one item or a query 4986 if not isinstance(items,list): 4987 #use a keys_only query to ensure that this runs as a datastore 4988 # small operations 4989 leftitems = items.fetch(1000, keys_only=True) 4990 counter = 0 4991 while len(leftitems): 4992 counter += len(leftitems) 4993 gae.delete(leftitems) 4994 leftitems = items.fetch(1000, keys_only=True) 4995 else: 4996 counter = len(items) 4997 gae.delete(items) 4998 return counter
4999
5000 - def update(self,tablename,query,update_fields):
5001 # self.db['_lastsql'] = self._update(tablename,query,update_fields) 5002 (items, tablename, fields) = self.select_raw(query) 5003 counter = 0 5004 for item in items: 5005 for field, value in update_fields: 5006 setattr(item, field.name, self.represent(value,field.type)) 5007 item.put() 5008 counter += 1 5009 LOGGER.info(str(counter)) 5010 return counter
5011
5012 - def insert(self,table,fields):
5013 dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields) 5014 # table._db['_lastsql'] = self._insert(table,fields) 5015 tmp = table._tableobj(**dfields) 5016 tmp.put() 5017 rid = Reference(tmp.key().id()) 5018 (rid._table, rid._record, rid._gaekey) = (table, None, tmp.key()) 5019 return rid
5020
5021 - def bulk_insert(self,table,items):
5022 parsed_items = [] 5023 for item in items: 5024 dfields=dict((f.name,self.represent(v,f.type)) for f,v in item) 5025 parsed_items.append(table._tableobj(**dfields)) 5026 gae.put(parsed_items) 5027 return True
5028
5029 -def uuid2int(uuidv):
5030 return uuid.UUID(uuidv).int
5031
5032 -def int2uuid(n):
5033 return str(uuid.UUID(int=n))
5034
5035 -class CouchDBAdapter(NoSQLAdapter):
5036 drivers = ('couchdb',) 5037 5038 uploads_in_blob = True 5039 types = { 5040 'boolean': bool, 5041 'string': str, 5042 'text': str, 5043 'json': str, 5044 'password': str, 5045 'blob': str, 5046 'upload': str, 5047 'integer': long, 5048 'bigint': long, 5049 'float': float, 5050 'double': float, 5051 'date': datetime.date, 5052 'time': datetime.time, 5053 'datetime': datetime.datetime, 5054 'id': long, 5055 'reference': long, 5056 'list:string': list, 5057 'list:integer': list, 5058 'list:reference': list, 5059 } 5060
5061 - def file_exists(self, filename): pass
5062 - def file_open(self, filename, mode='rb', lock=True): pass
5063 - def file_close(self, fileobj): pass
5064
5065 - def expand(self,expression,field_type=None):
5066 if isinstance(expression,Field): 5067 if expression.type=='id': 5068 return "%s._id" % expression.tablename 5069 return BaseAdapter.expand(self,expression,field_type)
5070
5071 - def AND(self,first,second):
5072 return '(%s && %s)' % (self.expand(first),self.expand(second))
5073
5074 - def OR(self,first,second):
5075 return '(%s || %s)' % (self.expand(first),self.expand(second))
5076
5077 - def EQ(self,first,second):
5078 if second is None: 5079 return '(%s == null)' % self.expand(first) 5080 return '(%s == %s)' % (self.expand(first),self.expand(second,first.type))
5081
5082 - def NE(self,first,second):
5083 if second is None: 5084 return '(%s != null)' % self.expand(first) 5085 return '(%s != %s)' % (self.expand(first),self.expand(second,first.type))
5086
5087 - def COMMA(self,first,second):
5088 return '%s + %s' % (self.expand(first),self.expand(second))
5089
5090 - def represent(self, obj, fieldtype):
5091 value = NoSQLAdapter.represent(self, obj, fieldtype) 5092 if fieldtype=='id': 5093 return repr(str(long(value))) 5094 elif fieldtype in ('date','time','datetime','boolean'): 5095 return serializers.json(value) 5096 return repr(not isinstance(value,unicode) and value \ 5097 or value and value.encode('utf8'))
5098
5099 - def __init__(self,db,uri='couchdb://127.0.0.1:5984', 5100 pool_size=0,folder=None,db_codec ='UTF-8', 5101 credential_decoder=IDENTITY, driver_args={}, 5102 adapter_args={}, do_connect=True, after_connection=None):
5103 self.db = db 5104 self.uri = uri 5105 if do_connect: self.find_driver(adapter_args) 5106 self.dbengine = 'couchdb' 5107 self.folder = folder 5108 db['_lastsql'] = '' 5109 self.db_codec = 'UTF-8' 5110 self._after_connection = after_connection 5111 self.pool_size = pool_size 5112 5113 url='http://'+uri[10:] 5114 def connector(url=url,driver_args=driver_args): 5115 return self.driver.Server(url,**driver_args)
5116 self.reconnect(connector,cursor=False)
5117
5118 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
5119 if migrate: 5120 try: 5121 self.connection.create(table._tablename) 5122 except: 5123 pass
5124
5125 - def insert(self,table,fields):
5126 id = uuid2int(web2py_uuid()) 5127 ctable = self.connection[table._tablename] 5128 values = dict((k.name,self.represent(v,k.type)) for k,v in fields) 5129 values['_id'] = str(id) 5130 ctable.save(values) 5131 return id
5132
5133 - def _select(self,query,fields,attributes):
5134 if not isinstance(query,Query): 5135 raise SyntaxError("Not Supported") 5136 for key in set(attributes.keys())-SELECT_ARGS: 5137 raise SyntaxError('invalid select attribute: %s' % key) 5138 new_fields=[] 5139 for item in fields: 5140 if isinstance(item,SQLALL): 5141 new_fields += item._table 5142 else: 5143 new_fields.append(item) 5144 def uid(fd): 5145 return fd=='id' and '_id' or fd
5146 def get(row,fd): 5147 return fd=='id' and long(row['_id']) or row.get(fd,None) 5148 fields = new_fields 5149 tablename = self.get_table(query) 5150 fieldnames = [f.name for f in (fields or self.db[tablename])] 5151 colnames = ['%s.%s' % (tablename,k) for k in fieldnames] 5152 fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames]) 5153 fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\ 5154 dict(t=tablename, 5155 query=self.expand(query), 5156 order='%s._id' % tablename, 5157 fields=fields) 5158 return fn, colnames 5159
5160 - def select(self,query,fields,attributes):
5161 if not isinstance(query,Query): 5162 raise SyntaxError("Not Supported") 5163 fn, colnames = self._select(query,fields,attributes) 5164 tablename = colnames[0].split('.')[0] 5165 ctable = self.connection[tablename] 5166 rows = [cols['value'] for cols in ctable.query(fn)] 5167 processor = attributes.get('processor',self.parse) 5168 return processor(rows,fields,colnames,False)
5169
5170 - def delete(self,tablename,query):
5171 if not isinstance(query,Query): 5172 raise SyntaxError("Not Supported") 5173 if query.first.type=='id' and query.op==self.EQ: 5174 id = query.second 5175 tablename = query.first.tablename 5176 assert(tablename == query.first.tablename) 5177 ctable = self.connection[tablename] 5178 try: 5179 del ctable[str(id)] 5180 return 1 5181 except couchdb.http.ResourceNotFound: 5182 return 0 5183 else: 5184 tablename = self.get_table(query) 5185 rows = self.select(query,[self.db[tablename]._id],{}) 5186 ctable = self.connection[tablename] 5187 for row in rows: 5188 del ctable[str(row.id)] 5189 return len(rows)
5190
5191 - def update(self,tablename,query,fields):
5192 if not isinstance(query,Query): 5193 raise SyntaxError("Not Supported") 5194 if query.first.type=='id' and query.op==self.EQ: 5195 id = query.second 5196 tablename = query.first.tablename 5197 ctable = self.connection[tablename] 5198 try: 5199 doc = ctable[str(id)] 5200 for key,value in fields: 5201 doc[key.name] = self.represent(value,self.db[tablename][key.name].type) 5202 ctable.save(doc) 5203 return 1 5204 except couchdb.http.ResourceNotFound: 5205 return 0 5206 else: 5207 tablename = self.get_table(query) 5208 rows = self.select(query,[self.db[tablename]._id],{}) 5209 ctable = self.connection[tablename] 5210 table = self.db[tablename] 5211 for row in rows: 5212 doc = ctable[str(row.id)] 5213 for key,value in fields: 5214 doc[key.name] = self.represent(value,table[key.name].type) 5215 ctable.save(doc) 5216 return len(rows)
5217
5218 - def count(self,query,distinct=None):
5219 if distinct: 5220 raise RuntimeError("COUNT DISTINCT not supported") 5221 if not isinstance(query,Query): 5222 raise SyntaxError("Not Supported") 5223 tablename = self.get_table(query) 5224 rows = self.select(query,[self.db[tablename]._id],{}) 5225 return len(rows)
5226
5227 -def cleanup(text):
5228 """ 5229 validates that the given text is clean: only contains [0-9a-zA-Z_] 5230 """ 5231 if not REGEX_ALPHANUMERIC.match(text): 5232 raise SyntaxError('invalid table or field name: %s' % text) 5233 return text
5234
5235 -class MongoDBAdapter(NoSQLAdapter):
5236 native_json = True 5237 drivers = ('pymongo',) 5238 5239 uploads_in_blob = True 5240 5241 types = { 5242 'boolean': bool, 5243 'string': str, 5244 'text': str, 5245 'json': str, 5246 'password': str, 5247 'blob': str, 5248 'upload': str, 5249 'integer': long, 5250 'bigint': long, 5251 'float': float, 5252 'double': float, 5253 'date': datetime.date, 5254 'time': datetime.time, 5255 'datetime': datetime.datetime, 5256 'id': long, 5257 'reference': long, 5258 'list:string': list, 5259 'list:integer': list, 5260 'list:reference': list, 5261 } 5262 5263 error_messages = {"javascript_needed": "This must yet be replaced" + 5264 " with javascript in order to work."} 5265
5266 - def __init__(self,db,uri='mongodb://127.0.0.1:5984/db', 5267 pool_size=0, folder=None, db_codec ='UTF-8', 5268 credential_decoder=IDENTITY, driver_args={}, 5269 adapter_args={}, do_connect=True, after_connection=None):
5270 5271 self.db = db 5272 self.uri = uri 5273 if do_connect: self.find_driver(adapter_args) 5274 import random 5275 from bson.objectid import ObjectId 5276 from bson.son import SON 5277 import pymongo.uri_parser 5278 5279 m = pymongo.uri_parser.parse_uri(uri) 5280 5281 self.SON = SON 5282 self.ObjectId = ObjectId 5283 self.random = random 5284 5285 self.dbengine = 'mongodb' 5286 self.folder = folder 5287 db['_lastsql'] = '' 5288 self.db_codec = 'UTF-8' 5289 self._after_connection = after_connection 5290 self.pool_size = pool_size 5291 #this is the minimum amount of replicates that it should wait 5292 # for on insert/update 5293 self.minimumreplication = adapter_args.get('minimumreplication',0) 5294 # by default all inserts and selects are performand asynchronous, 5295 # but now the default is 5296 # synchronous, except when overruled by either this default or 5297 # function parameter 5298 self.safe = adapter_args.get('safe',True) 5299 5300 if isinstance(m,tuple): 5301 m = {"database" : m[1]} 5302 if m.get('database')==None: 5303 raise SyntaxError("Database is required!") 5304 5305 def connector(uri=self.uri,m=m): 5306 # Connection() is deprecated 5307 if hasattr(self.driver, "MongoClient"): 5308 Connection = self.driver.MongoClient 5309 else: 5310 Connection = self.driver.Connection 5311 return Connection(uri)[m.get('database')]
5312 5313 self.reconnect(connector,cursor=False)
5314
5315 - def object_id(self, arg=None):
5316 """ Convert input to a valid Mongodb ObjectId instance 5317 5318 self.object_id("<random>") -> ObjectId (not unique) instance """ 5319 if not arg: 5320 arg = 0 5321 if isinstance(arg, basestring): 5322 # we assume an integer as default input 5323 rawhex = len(arg.replace("0x", "").replace("L", "")) == 24 5324 if arg.isdigit() and (not rawhex): 5325 arg = int(arg) 5326 elif arg == "<random>": 5327 arg = int("0x%sL" % \ 5328 "".join([self.random.choice("0123456789abcdef") \ 5329 for x in range(24)]), 0) 5330 elif arg.isalnum(): 5331 if not arg.startswith("0x"): 5332 arg = "0x%s" % arg 5333 try: 5334 arg = int(arg, 0) 5335 except ValueError, e: 5336 raise ValueError( 5337 "invalid objectid argument string: %s" % e) 5338 else: 5339 raise ValueError("Invalid objectid argument string. " + 5340 "Requires an integer or base 16 value") 5341 elif isinstance(arg, self.ObjectId): 5342 return arg 5343 5344 if not isinstance(arg, (int, long)): 5345 raise TypeError("object_id argument must be of type " + 5346 "ObjectId or an objectid representable integer") 5347 if arg == 0: 5348 hexvalue = "".zfill(24) 5349 else: 5350 hexvalue = hex(arg)[2:].replace("L", "") 5351 return self.ObjectId(hexvalue)
5352
5353 - def parse_reference(self, value, field_type):
5354 # here we have to check for ObjectID before base parse 5355 if isinstance(value, self.ObjectId): 5356 value = long(str(value), 16) 5357 return super(MongoDBAdapter, 5358 self).parse_reference(value, field_type)
5359
5360 - def parse_id(self, value, field_type):
5361 if isinstance(value, self.ObjectId): 5362 value = long(str(value), 16) 5363 return super(MongoDBAdapter, 5364 self).parse_id(value, field_type)
5365
5366 - def represent(self, obj, fieldtype):
5367 # the base adatpter does not support MongoDB ObjectId 5368 if isinstance(obj, self.ObjectId): 5369 value = obj 5370 else: 5371 value = NoSQLAdapter.represent(self, obj, fieldtype) 5372 # reference types must be convert to ObjectID 5373 if fieldtype =='date': 5374 if value == None: 5375 return value 5376 # this piece of data can be stripped off based on the fieldtype 5377 t = datetime.time(0, 0, 0) 5378 # mongodb doesn't has a date object and so it must datetime, 5379 # string or integer 5380 return datetime.datetime.combine(value, t) 5381 elif fieldtype == 'time': 5382 if value == None: 5383 return value 5384 # this piece of data can be stripped of based on the fieldtype 5385 d = datetime.date(2000, 1, 1) 5386 # mongodb doesn't has a time object and so it must datetime, 5387 # string or integer 5388 return datetime.datetime.combine(d, value) 5389 elif fieldtype == "blob": 5390 from bson import Binary 5391 if not isinstance(value, Binary): 5392 return Binary(value) 5393 return value 5394 elif (isinstance(fieldtype, basestring) and 5395 fieldtype.startswith('list:')): 5396 if fieldtype.startswith('list:reference'): 5397 newval = [] 5398 for v in value: 5399 newval.append(self.object_id(v)) 5400 return newval 5401 return value 5402 elif ((isinstance(fieldtype, basestring) and 5403 fieldtype.startswith("reference")) or 5404 (isinstance(fieldtype, Table)) or fieldtype=="id"): 5405 value = self.object_id(value) 5406 return value
5407
5408 - def create_table(self, table, migrate=True, fake_migrate=False, 5409 polymodel=None, isCapped=False):
5410 if isCapped: 5411 raise RuntimeError("Not implemented")
5412
5413 - def count(self, query, distinct=None, snapshot=True):
5414 if distinct: 5415 raise RuntimeError("COUNT DISTINCT not supported") 5416 if not isinstance(query,Query): 5417 raise SyntaxError("Not Supported") 5418 tablename = self.get_table(query) 5419 return long(self.select(query,[self.db[tablename]._id], {}, 5420 count=True,snapshot=snapshot)['count'])
5421 # Maybe it would be faster if we just implemented the pymongo 5422 # .count() function which is probably quicker? 5423 # therefor call __select() connection[table].find(query).count() 5424 # Since this will probably reduce the return set? 5425
5426 - def expand(self, expression, field_type=None):
5427 if isinstance(expression, Query): 5428 # any query using 'id':= 5429 # set name as _id (as per pymongo/mongodb primary key) 5430 # convert second arg to an objectid field 5431 # (if its not already) 5432 # if second arg is 0 convert to objectid 5433 if isinstance(expression.first,Field) and \ 5434 ((expression.first.type == 'id') or \ 5435 ("reference" in expression.first.type)): 5436 if expression.first.type == 'id': 5437 expression.first.name = '_id' 5438 # cast to Mongo ObjectId 5439 if isinstance(expression.second, (tuple, list, set)): 5440 expression.second = [self.object_id(item) for 5441 item in expression.second] 5442 else: 5443 expression.second = self.object_id(expression.second) 5444 result = expression.op(expression.first, expression.second) 5445 5446 if isinstance(expression, Field): 5447 if expression.type=='id': 5448 result = "_id" 5449 else: 5450 result = expression.name 5451 elif isinstance(expression, (Expression, Query)): 5452 if not expression.second is None: 5453 result = expression.op(expression.first, expression.second) 5454 elif not expression.first is None: 5455 result = expression.op(expression.first) 5456 elif not isinstance(expression.op, str): 5457 result = expression.op() 5458 else: 5459 result = expression.op 5460 elif field_type: 5461 result = self.represent(expression,field_type) 5462 elif isinstance(expression,(list,tuple)): 5463 result = ','.join(self.represent(item,field_type) for 5464 item in expression) 5465 else: 5466 result = expression 5467 return result
5468
5469 - def drop(self, table, mode=''):
5470 ctable = self.connection[table._tablename] 5471 ctable.drop()
5472
5473 - def truncate(self, table, mode, safe=None):
5474 if safe == None: 5475 safe=self.safe 5476 ctable = self.connection[table._tablename] 5477 ctable.remove(None, safe=True)
5478
5479 - def _select(self, query, fields, attributes):
5480 if 'for_update' in attributes: 5481 logging.warn('mongodb does not support for_update') 5482 for key in set(attributes.keys())-set(('limitby', 5483 'orderby','for_update')): 5484 if attributes[key]!=None: 5485 logging.warn('select attribute not implemented: %s' % key) 5486 5487 new_fields=[] 5488 mongosort_list = [] 5489 5490 # try an orderby attribute 5491 orderby = attributes.get('orderby', False) 5492 limitby = attributes.get('limitby', False) 5493 # distinct = attributes.get('distinct', False) 5494 if orderby: 5495 if isinstance(orderby, (list, tuple)): 5496 orderby = xorify(orderby) 5497 5498 # !!!! need to add 'random' 5499 for f in self.expand(orderby).split(','): 5500 if f.startswith('-'): 5501 mongosort_list.append((f[1:], -1)) 5502 else: 5503 mongosort_list.append((f, 1)) 5504 if limitby: 5505 limitby_skip, limitby_limit = limitby[0], int(limitby[1]) 5506 else: 5507 limitby_skip = limitby_limit = 0 5508 5509 mongofields_dict = self.SON() 5510 mongoqry_dict = {} 5511 for item in fields: 5512 if isinstance(item, SQLALL): 5513 new_fields += item._table 5514 else: 5515 new_fields.append(item) 5516 fields = new_fields 5517 if isinstance(query,Query): 5518 tablename = self.get_table(query) 5519 elif len(fields) != 0: 5520 tablename = fields[0].tablename 5521 else: 5522 raise SyntaxError("The table name could not be found in " + 5523 "the query nor from the select statement.") 5524 mongoqry_dict = self.expand(query) 5525 fields = fields or self.db[tablename] 5526 for field in fields: 5527 mongofields_dict[field.name] = 1 5528 5529 return tablename, mongoqry_dict, mongofields_dict, mongosort_list, \ 5530 limitby_limit, limitby_skip
5531
5532 - def select(self, query, fields, attributes, count=False, 5533 snapshot=False):
5534 # TODO: support joins 5535 tablename, mongoqry_dict, mongofields_dict, mongosort_list, \ 5536 limitby_limit, limitby_skip = self._select(query, fields, attributes) 5537 ctable = self.connection[tablename] 5538 5539 if count: 5540 return {'count' : ctable.find( 5541 mongoqry_dict, mongofields_dict, 5542 skip=limitby_skip, limit=limitby_limit, 5543 sort=mongosort_list, snapshot=snapshot).count()} 5544 else: 5545 # pymongo cursor object 5546 mongo_list_dicts = ctable.find(mongoqry_dict, 5547 mongofields_dict, skip=limitby_skip, 5548 limit=limitby_limit, sort=mongosort_list, 5549 snapshot=snapshot) 5550 rows = [] 5551 # populate row in proper order 5552 # Here we replace ._id with .id to follow the standard naming 5553 colnames = [] 5554 newnames = [] 5555 for field in fields: 5556 colname = str(field) 5557 colnames.append(colname) 5558 tablename, fieldname = colname.split(".") 5559 if fieldname == "_id": 5560 # Mongodb reserved uuid key 5561 field.name = "id" 5562 newnames.append(".".join((tablename, field.name))) 5563 5564 for record in mongo_list_dicts: 5565 row=[] 5566 for colname in colnames: 5567 tablename, fieldname = colname.split(".") 5568 # switch to Mongo _id uuids for retrieving 5569 # record id's 5570 if fieldname == "id": fieldname = "_id" 5571 if fieldname in record: 5572 value = record[fieldname] 5573 else: 5574 value = None 5575 row.append(value) 5576 rows.append(row) 5577 5578 processor = attributes.get('processor', self.parse) 5579 result = processor(rows, fields, newnames, False) 5580 return result
5581
5582 - def _insert(self, table, fields):
5583 values = dict() 5584 for k, v in fields: 5585 if not k.name in ["id", "safe"]: 5586 fieldname = k.name 5587 fieldtype = table[k.name].type 5588 values[fieldname] = self.represent(v, fieldtype) 5589 return values
5590 5591 # Safe determines whether a asynchronious request is done or a 5592 # synchronious action is done 5593 # For safety, we use by default synchronous requests
5594 - def insert(self, table, fields, safe=None):
5595 if safe==None: 5596 safe = self.safe 5597 ctable = self.connection[table._tablename] 5598 values = self._insert(table, fields) 5599 ctable.insert(values, safe=safe) 5600 return long(str(values['_id']), 16)
5601 5602 #this function returns a dict with the where clause and update fields
5603 - def _update(self, tablename, query, fields):
5604 if not isinstance(query, Query): 5605 raise SyntaxError("Not Supported") 5606 filter = None 5607 if query: 5608 filter = self.expand(query) 5609 # do not try to update id fields to avoid backend errors 5610 modify = {'$set': dict((k.name, self.represent(v, k.type)) for 5611 k, v in fields if (not k.name in ("_id", "id")))} 5612 return modify, filter
5613
5614 - def update(self, tablename, query, fields, safe=None):
5615 if safe == None: 5616 safe = self.safe 5617 # return amount of adjusted rows or zero, but no exceptions 5618 # @ related not finding the result 5619 if not isinstance(query, Query): 5620 raise RuntimeError("Not implemented") 5621 amount = self.count(query, False) 5622 modify, filter = self._update(tablename, query, fields) 5623 try: 5624 result = self.connection[tablename].update(filter, 5625 modify, multi=True, safe=safe) 5626 if safe: 5627 try: 5628 # if result count is available fetch it 5629 return result["n"] 5630 except (KeyError, AttributeError, TypeError): 5631 return amount 5632 else: 5633 return amount 5634 except Exception, e: 5635 # TODO Reverse update query to verifiy that the query succeded 5636 raise RuntimeError("uncaught exception when updating rows: %s" % e)
5637
5638 - def _delete(self, tablename, query):
5639 if not isinstance(query, Query): 5640 raise RuntimeError("query type %s is not supported" % \ 5641 type(query)) 5642 return self.expand(query)
5643
5644 - def delete(self, tablename, query, safe=None):
5645 if safe is None: 5646 safe = self.safe 5647 amount = 0 5648 amount = self.count(query, False) 5649 filter = self._delete(tablename, query) 5650 self.connection[tablename].remove(filter, safe=safe) 5651 return amount
5652
5653 - def bulk_insert(self, table, items):
5654 return [self.insert(table,item) for item in items]
5655 5656 ## OPERATORS
5657 - def INVERT(self, first):
5658 #print "in invert first=%s" % first 5659 return '-%s' % self.expand(first)
5660 5661 # TODO This will probably not work:(
5662 - def NOT(self, first):
5663 result = {} 5664 result["$not"] = self.expand(first) 5665 return result
5666
5667 - def AND(self,first,second):
5668 f = self.expand(first) 5669 s = self.expand(second) 5670 f.update(s) 5671 return f
5672
5673 - def OR(self,first,second):
5674 # pymongo expects: .find({'$or': [{'name':'1'}, {'name':'2'}]}) 5675 result = {} 5676 f = self.expand(first) 5677 s = self.expand(second) 5678 result['$or'] = [f,s] 5679 return result
5680
5681 - def BELONGS(self, first, second):
5682 if isinstance(second, str): 5683 return {self.expand(first) : {"$in" : [ second[:-1]]} } 5684 elif second==[] or second==() or second==set(): 5685 return {1:0} 5686 items = [self.expand(item, first.type) for item in second] 5687 return {self.expand(first) : {"$in" : items} }
5688
5689 - def EQ(self,first,second=None):
5690 result = {} 5691 result[self.expand(first)] = self.expand(second) 5692 return result
5693
5694 - def NE(self, first, second=None):
5695 result = {} 5696 result[self.expand(first)] = {'$ne': self.expand(second)} 5697 return result
5698
5699 - def LT(self,first,second=None):
5700 if second is None: 5701 raise RuntimeError("Cannot compare %s < None" % first) 5702 result = {} 5703 result[self.expand(first)] = {'$lt': self.expand(second)} 5704 return result
5705
5706 - def LE(self,first,second=None):
5707 if second is None: 5708 raise RuntimeError("Cannot compare %s <= None" % first) 5709 result = {} 5710 result[self.expand(first)] = {'$lte': self.expand(second)} 5711 return result
5712
5713 - def GT(self,first,second):
5714 result = {} 5715 result[self.expand(first)] = {'$gt': self.expand(second)} 5716 return result
5717
5718 - def GE(self,first,second=None):
5719 if second is None: 5720 raise RuntimeError("Cannot compare %s >= None" % first) 5721 result = {} 5722 result[self.expand(first)] = {'$gte': self.expand(second)} 5723 return result
5724
5725 - def ADD(self, first, second):
5726 raise NotImplementedError(self.error_messages["javascript_needed"]) 5727 return '%s + %s' % (self.expand(first), 5728 self.expand(second, first.type))
5729
5730 - def SUB(self, first, second):
5731 raise NotImplementedError(self.error_messages["javascript_needed"]) 5732 return '(%s - %s)' % (self.expand(first), 5733 self.expand(second, first.type))
5734
5735 - def MUL(self, first, second):
5736 raise NotImplementedError(self.error_messages["javascript_needed"]) 5737 return '(%s * %s)' % (self.expand(first), 5738 self.expand(second, first.type))
5739
5740 - def DIV(self, first, second):
5741 raise NotImplementedError(self.error_messages["javascript_needed"]) 5742 return '(%s / %s)' % (self.expand(first), 5743 self.expand(second, first.type))
5744
5745 - def MOD(self, first, second):
5746 raise NotImplementedError(self.error_messages["javascript_needed"]) 5747 return '(%s %% %s)' % (self.expand(first), 5748 self.expand(second, first.type))
5749
5750 - def AS(self, first, second):
5751 raise NotImplementedError(self.error_messages["javascript_needed"]) 5752 return '%s AS %s' % (self.expand(first), second)
5753 5754 # We could implement an option that simulates a full featured SQL 5755 # database. But I think the option should be set explicit or 5756 # implemented as another library.
5757 - def ON(self, first, second):
5758 raise NotImplementedError("This is not possible in NoSQL" + 5759 " but can be simulated with a wrapper.") 5760 return '%s ON %s' % (self.expand(first), self.expand(second))
5761 5762 # BLOW ARE TWO IMPLEMENTATIONS OF THE SAME FUNCITONS 5763 # WHICH ONE IS BEST? 5764
5765 - def COMMA(self, first, second):
5766 return '%s, %s' % (self.expand(first), self.expand(second))
5767
5768 - def LIKE(self, first, second):
5769 #escaping regex operators? 5770 return {self.expand(first): ('%s' % \ 5771 self.expand(second, 'string').replace('%','/'))}
5772
5773 - def STARTSWITH(self, first, second):
5774 #escaping regex operators? 5775 return {self.expand(first): ('/^%s/' % \ 5776 self.expand(second, 'string'))}
5777
5778 - def ENDSWITH(self, first, second):
5779 #escaping regex operators? 5780 return {self.expand(first): ('/%s^/' % \ 5781 self.expand(second, 'string'))}
5782
5783 - def CONTAINS(self, first, second, case_sensitive=False):
5784 # silently ignore, only case sensitive 5785 # There is a technical difference, but mongodb doesn't support 5786 # that, but the result will be the same 5787 val = second if isinstance(second,self.ObjectId) else \ 5788 {'$regex':".*" + re.escape(self.expand(second, 'string')) + ".*"} 5789 return {self.expand(first) : val}
5790
5791 - def LIKE(self, first, second):
5792 import re 5793 return {self.expand(first): {'$regex': \ 5794 re.escape(self.expand(second, 5795 'string')).replace('%','.*')}}
5796 5797 #TODO verify full compatibilty with official SQL Like operator
5798 - def STARTSWITH(self, first, second):
5799 #TODO Solve almost the same problem as with endswith 5800 import re 5801 return {self.expand(first): {'$regex' : '^' + 5802 re.escape(self.expand(second, 5803 'string'))}}
5804 5805 #TODO verify full compatibilty with official SQL Like operator
5806 - def ENDSWITH(self, first, second):
5807 #escaping regex operators? 5808 #TODO if searched for a name like zsa_corbitt and the function 5809 # is endswith('a') then this is also returned. 5810 # Aldo it end with a t 5811 import re 5812 return {self.expand(first): {'$regex': \ 5813 re.escape(self.expand(second, 'string')) + '$'}}
5814 5815 #TODO verify full compatibilty with official oracle contains operator
5816 - def CONTAINS(self, first, second, case_sensitive=False):
5817 # silently ignore, only case sensitive 5818 #There is a technical difference, but mongodb doesn't support 5819 # that, but the result will be the same 5820 #TODO contains operators need to be transformed to Regex 5821 return {self.expand(first) : {'$regex': \ 5822 ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
5823
5824 5825 -class IMAPAdapter(NoSQLAdapter):
5826 drivers = ('imaplib',) 5827 5828 """ IMAP server adapter 5829 5830 This class is intended as an interface with 5831 email IMAP servers to perform simple queries in the 5832 web2py DAL query syntax, so email read, search and 5833 other related IMAP mail services (as those implemented 5834 by brands like Google(r), and Yahoo!(r) 5835 can be managed from web2py applications. 5836 5837 The code uses examples by Yuji Tomita on this post: 5838 http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137 5839 and is based in docs for Python imaplib, python email 5840 and email IETF's (i.e. RFC2060 and RFC3501) 5841 5842 This adapter was tested with a small set of operations with Gmail(r). Other 5843 services requests could raise command syntax and response data issues. 5844 5845 It creates its table and field names "statically", 5846 meaning that the developer should leave the table and field 5847 definitions to the DAL instance by calling the adapter's 5848 .define_tables() method. The tables are defined with the 5849 IMAP server mailbox list information. 5850 5851 .define_tables() returns a dictionary mapping dal tablenames 5852 to the server mailbox names with the following structure: 5853 5854 {<tablename>: str <server mailbox name>} 5855 5856 Here is a list of supported fields: 5857 5858 Field Type Description 5859 ################################################################ 5860 uid string 5861 answered boolean Flag 5862 created date 5863 content list:string A list of dict text or html parts 5864 to string 5865 cc string 5866 bcc string 5867 size integer the amount of octets of the message* 5868 deleted boolean Flag 5869 draft boolean Flag 5870 flagged boolean Flag 5871 sender string 5872 recent boolean Flag 5873 seen boolean Flag 5874 subject string 5875 mime string The mime header declaration 5876 email string The complete RFC822 message** 5877 attachments <type list> Each non text part as dict 5878 encoding string The main detected encoding 5879 5880 *At the application side it is measured as the length of the RFC822 5881 message string 5882 5883 WARNING: As row id's are mapped to email sequence numbers, 5884 make sure your imap client web2py app does not delete messages 5885 during select or update actions, to prevent 5886 updating or deleting different messages. 5887 Sequence numbers change whenever the mailbox is updated. 5888 To avoid this sequence numbers issues, it is recommended the use 5889 of uid fields in query references (although the update and delete 5890 in separate actions rule still applies). 5891 5892 # This is the code recommended to start imap support 5893 # at the app's model: 5894 5895 imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl 5896 imapdb.define_tables() 5897 5898 Here is an (incomplete) list of possible imap commands: 5899 5900 # Count today's unseen messages 5901 # smaller than 6000 octets from the 5902 # inbox mailbox 5903 5904 q = imapdb.INBOX.seen == False 5905 q &= imapdb.INBOX.created == datetime.date.today() 5906 q &= imapdb.INBOX.size < 6000 5907 unread = imapdb(q).count() 5908 5909 # Fetch last query messages 5910 rows = imapdb(q).select() 5911 5912 # it is also possible to filter query select results with limitby and 5913 # sequences of mailbox fields 5914 5915 set.select(<fields sequence>, limitby=(<int>, <int>)) 5916 5917 # Mark last query messages as seen 5918 messages = [row.uid for row in rows] 5919 seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True) 5920 5921 # Delete messages in the imap database that have mails from mr. Gumby 5922 5923 deleted = 0 5924 for mailbox in imapdb.tables 5925 deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete() 5926 5927 # It is possible also to mark messages for deletion instead of ereasing them 5928 # directly with set.update(deleted=True) 5929 5930 5931 # This object give access 5932 # to the adapter auto mailbox 5933 # mapped names (which native 5934 # mailbox has what table name) 5935 5936 imapdb.mailboxes <dict> # tablename, server native name pairs 5937 5938 # To retrieve a table native mailbox name use: 5939 imapdb.<table>.mailbox 5940 5941 ### New features v2.4.1: 5942 5943 # Declare mailboxes statically with tablename, name pairs 5944 # This avoids the extra server names retrieval 5945 5946 imapdb.define_tables({"inbox": "INBOX"}) 5947 5948 # Selects without content/attachments/email columns will only 5949 # fetch header and flags 5950 5951 imapdb(q).select(imapdb.INBOX.sender, imapdb.INBOX.subject) 5952 """ 5953 5954 types = { 5955 'string': str, 5956 'text': str, 5957 'date': datetime.date, 5958 'datetime': datetime.datetime, 5959 'id': long, 5960 'boolean': bool, 5961 'integer': int, 5962 'bigint': long, 5963 'blob': str, 5964 'list:string': str, 5965 } 5966 5967 dbengine = 'imap' 5968 5969 REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$') 5970
5971 - def __init__(self, 5972 db, 5973 uri, 5974 pool_size=0, 5975 folder=None, 5976 db_codec ='UTF-8', 5977 credential_decoder=IDENTITY, 5978 driver_args={}, 5979 adapter_args={}, 5980 do_connect=True, 5981 after_connection=None):
5982 5983 # db uri: user@example.com:password@imap.server.com:123 5984 # TODO: max size adapter argument for preventing large mail transfers 5985 5986 self.db = db 5987 self.uri = uri 5988 if do_connect: self.find_driver(adapter_args) 5989 self.pool_size=pool_size 5990 self.folder = folder 5991 self.db_codec = db_codec 5992 self._after_connection = after_connection 5993 self.credential_decoder = credential_decoder 5994 self.driver_args = driver_args 5995 self.adapter_args = adapter_args 5996 self.mailbox_size = None 5997 self.static_names = None 5998 self.charset = sys.getfilesystemencoding() 5999 # imap class 6000 self.imap4 = None 6001 uri = uri.split("://")[1] 6002 6003 """ MESSAGE is an identifier for sequence number""" 6004 6005 self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft', 6006 'flagged': '\\Flagged', 'recent': '\\Recent', 6007 'seen': '\\Seen', 'answered': '\\Answered'} 6008 self.search_fields = { 6009 'id': 'MESSAGE', 'created': 'DATE', 6010 'uid': 'UID', 'sender': 'FROM', 6011 'to': 'TO', 'cc': 'CC', 6012 'bcc': 'BCC', 'content': 'TEXT', 6013 'size': 'SIZE', 'deleted': '\\Deleted', 6014 'draft': '\\Draft', 'flagged': '\\Flagged', 6015 'recent': '\\Recent', 'seen': '\\Seen', 6016 'subject': 'SUBJECT', 'answered': '\\Answered', 6017 'mime': None, 'email': None, 6018 'attachments': None 6019 } 6020 6021 db['_lastsql'] = '' 6022 6023 m = self.REGEX_URI.match(uri) 6024 user = m.group('user') 6025 password = m.group('password') 6026 host = m.group('host') 6027 port = int(m.group('port')) 6028 over_ssl = False 6029 if port==993: 6030 over_ssl = True 6031 6032 driver_args.update(host=host,port=port, password=password, user=user) 6033 def connector(driver_args=driver_args): 6034 # it is assumed sucessful authentication alLways 6035 # TODO: support direct connection and login tests 6036 if over_ssl: 6037 self.imap4 = self.driver.IMAP4_SSL 6038 else: 6039 self.imap4 = self.driver.IMAP4 6040 connection = self.imap4(driver_args["host"], driver_args["port"]) 6041 data = connection.login(driver_args["user"], driver_args["password"]) 6042 6043 # static mailbox list 6044 connection.mailbox_names = None 6045 6046 # dummy cursor function 6047 connection.cursor = lambda : True 6048 6049 return connection
6050 6051 self.db.define_tables = self.define_tables 6052 self.connector = connector 6053 if do_connect: self.reconnect()
6054
6055 - def reconnect(self, f=None, cursor=True):
6056 """ 6057 IMAP4 Pool connection method 6058 6059 imap connection lacks of self cursor command. 6060 A custom command should be provided as a replacement 6061 for connection pooling to prevent uncaught remote session 6062 closing 6063 6064 """ 6065 if getattr(self,'connection',None) != None: 6066 return 6067 if f is None: 6068 f = self.connector 6069 6070 if not self.pool_size: 6071 self.connection = f() 6072 self.cursor = cursor and self.connection.cursor() 6073 else: 6074 POOLS = ConnectionPool.POOLS 6075 uri = self.uri 6076 while True: 6077 GLOBAL_LOCKER.acquire() 6078 if not uri in POOLS: 6079 POOLS[uri] = [] 6080 if POOLS[uri]: 6081 self.connection = POOLS[uri].pop() 6082 GLOBAL_LOCKER.release() 6083 self.cursor = cursor and self.connection.cursor() 6084 if self.cursor and self.check_active_connection: 6085 try: 6086 # check if connection is alive or close it 6087 result, data = self.connection.list() 6088 except: 6089 # Possible connection reset error 6090 # TODO: read exception class 6091 self.connection = f() 6092 break 6093 else: 6094 GLOBAL_LOCKER.release() 6095 self.connection = f() 6096 self.cursor = cursor and self.connection.cursor() 6097 break 6098 self.after_connection_hook()
6099
6100 - def get_last_message(self, tablename):
6101 last_message = None 6102 # request mailbox list to the server 6103 # if needed 6104 if not isinstance(self.connection.mailbox_names, dict): 6105 self.get_mailboxes() 6106 try: 6107 result = self.connection.select(self.connection.mailbox_names[tablename]) 6108 last_message = int(result[1][0]) 6109 except (IndexError, ValueError, TypeError, KeyError): 6110 e = sys.exc_info()[1] 6111 LOGGER.debug("Error retrieving the last mailbox sequence number. %s" % str(e)) 6112 return last_message
6113
6114 - def get_uid_bounds(self, tablename):
6115 if not isinstance(self.connection.mailbox_names, dict): 6116 self.get_mailboxes() 6117 # fetch first and last messages 6118 # return (first, last) messages uid's 6119 last_message = self.get_last_message(tablename) 6120 result, data = self.connection.uid("search", None, "(ALL)") 6121 uid_list = data[0].strip().split() 6122 if len(uid_list) <= 0: 6123 return None 6124 else: 6125 return (uid_list[0], uid_list[-1])
6126
6127 - def convert_date(self, date, add=None):
6128 if add is None: 6129 add = datetime.timedelta() 6130 """ Convert a date object to a string 6131 with d-Mon-Y style for IMAP or the inverse 6132 case 6133 6134 add <timedelta> adds to the date object 6135 """ 6136 months = [None, "JAN","FEB","MAR","APR","MAY","JUN", 6137 "JUL", "AUG","SEP","OCT","NOV","DEC"] 6138 if isinstance(date, basestring): 6139 # Prevent unexpected date response format 6140 try: 6141 dayname, datestring = date.split(",") 6142 date_list = datestring.strip().split() 6143 year = int(date_list[2]) 6144 month = months.index(date_list[1].upper()) 6145 day = int(date_list[0]) 6146 hms = map(int, date_list[3].split(":")) 6147 return datetime.datetime(year, month, day, 6148 hms[0], hms[1], hms[2]) + add 6149 except (ValueError, AttributeError, IndexError), e: 6150 LOGGER.error("Could not parse date text: %s. %s" % 6151 (date, e)) 6152 return None 6153 elif isinstance(date, (datetime.datetime, datetime.date)): 6154 return (date + add).strftime("%d-%b-%Y") 6155 else: 6156 return None
6157 6158 @staticmethod
6159 - def header_represent(f, r):
6160 from email.header import decode_header 6161 text, encoding = decode_header(f)[0] 6162 if encoding: 6163 text = text.decode(encoding).encode('utf-8') 6164 return text
6165
6166 - def encode_text(self, text, charset, errors="replace"):
6167 """ convert text for mail to unicode""" 6168 if text is None: 6169 text = "" 6170 else: 6171 if isinstance(text, str): 6172 if charset is None: 6173 text = unicode(text, "utf-8", errors) 6174 else: 6175 text = unicode(text, charset, errors) 6176 else: 6177 raise Exception("Unsupported mail text type %s" % type(text)) 6178 return text.encode("utf-8")
6179
6180 - def get_charset(self, message):
6181 charset = message.get_content_charset() 6182 return charset
6183
6184 - def get_mailboxes(self):
6185 """ Query the mail database for mailbox names """ 6186 if self.static_names: 6187 # statically defined mailbox names 6188 self.connection.mailbox_names = self.static_names 6189 return self.static_names.keys() 6190 6191 mailboxes_list = self.connection.list() 6192 self.connection.mailbox_names = dict() 6193 mailboxes = list() 6194 x = 0 6195 for item in mailboxes_list[1]: 6196 x = x + 1 6197 item = item.strip() 6198 if not "NOSELECT" in item.upper(): 6199 sub_items = item.split("\"") 6200 sub_items = [sub_item for sub_item in sub_items \ 6201 if len(sub_item.strip()) > 0] 6202 # mailbox = sub_items[len(sub_items) -1] 6203 mailbox = sub_items[-1] 6204 # remove unwanted characters and store original names 6205 # Don't allow leading non alphabetic characters 6206 mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox))) 6207 mailboxes.append(mailbox_name) 6208 self.connection.mailbox_names[mailbox_name] = mailbox 6209 6210 return mailboxes
6211
6212 - def get_query_mailbox(self, query):
6213 nofield = True 6214 tablename = None 6215 attr = query 6216 while nofield: 6217 if hasattr(attr, "first"): 6218 attr = attr.first 6219 if isinstance(attr, Field): 6220 return attr.tablename 6221 elif isinstance(attr, Query): 6222 pass 6223 else: 6224 return None 6225 else: 6226 return None 6227 return tablename
6228
6229 - def is_flag(self, flag):
6230 if self.search_fields.get(flag, None) in self.flags.values(): 6231 return True 6232 else: 6233 return False
6234
6235 - def define_tables(self, mailbox_names=None):
6236 """ 6237 Auto create common IMAP fileds 6238 6239 This function creates fields definitions "statically" 6240 meaning that custom fields as in other adapters should 6241 not be supported and definitions handled on a service/mode 6242 basis (local syntax for Gmail(r), Ymail(r) 6243 6244 Returns a dictionary with tablename, server native mailbox name 6245 pairs. 6246 """ 6247 if mailbox_names: 6248 # optional statically declared mailboxes 6249 self.static_names = mailbox_names 6250 else: 6251 self.static_names = None 6252 if not isinstance(self.connection.mailbox_names, dict): 6253 self.get_mailboxes() 6254 6255 names = self.connection.mailbox_names.keys() 6256 6257 for name in names: 6258 self.db.define_table("%s" % name, 6259 Field("uid", "string", writable=False), 6260 Field("answered", "boolean"), 6261 Field("created", "datetime", writable=False), 6262 Field("content", list, writable=False), 6263 Field("to", "string", writable=False), 6264 Field("cc", "string", writable=False), 6265 Field("bcc", "string", writable=False), 6266 Field("size", "integer", writable=False), 6267 Field("deleted", "boolean"), 6268 Field("draft", "boolean"), 6269 Field("flagged", "boolean"), 6270 Field("sender", "string", writable=False), 6271 Field("recent", "boolean", writable=False), 6272 Field("seen", "boolean"), 6273 Field("subject", "string", writable=False), 6274 Field("mime", "string", writable=False), 6275 Field("email", "string", writable=False, readable=False), 6276 Field("attachments", list, writable=False, readable=False), 6277 Field("encoding", writable=False) 6278 ) 6279 6280 # Set a special _mailbox attribute for storing 6281 # native mailbox names 6282 self.db[name].mailbox = \ 6283 self.connection.mailbox_names[name] 6284 6285 # decode quoted printable 6286 self.db[name].to.represent = self.db[name].cc.represent = \ 6287 self.db[name].bcc.represent = self.db[name].sender.represent = \ 6288 self.db[name].subject.represent = self.header_represent 6289 6290 # Set the db instance mailbox collections 6291 self.db.mailboxes = self.connection.mailbox_names 6292 return self.db.mailboxes
6293
6294 - def create_table(self, *args, **kwargs):
6295 # not implemented 6296 # but required by DAL 6297 pass
6298
6299 - def _select(self, query, fields, attributes):
6300 if use_common_filters(query): 6301 query = self.common_filter(query, [self.get_query_mailbox(query),]) 6302 return str(query)
6303
6304 - def select(self, query, fields, attributes):
6305 """ Search and Fetch records and return web2py rows 6306 """ 6307 # move this statement elsewhere (upper-level) 6308 if use_common_filters(query): 6309 query = self.common_filter(query, [self.get_query_mailbox(query),]) 6310 6311 import email 6312 # get records from imap server with search + fetch 6313 # convert results to a dictionary 6314 tablename = None 6315 fetch_results = list() 6316 6317 if isinstance(query, Query): 6318 tablename = self.get_table(query) 6319 mailbox = self.connection.mailbox_names.get(tablename, None) 6320 if mailbox is None: 6321 raise ValueError("Mailbox name not found: %s" % mailbox) 6322 else: 6323 # select with readonly 6324 result, selected = self.connection.select(mailbox, True) 6325 if result != "OK": 6326 raise Exception("IMAP error: %s" % selected) 6327 self.mailbox_size = int(selected[0]) 6328 search_query = "(%s)" % str(query).strip() 6329 search_result = self.connection.uid("search", None, search_query) 6330 # Normal IMAP response OK is assumed (change this) 6331 if search_result[0] == "OK": 6332 # For "light" remote server responses just get the first 6333 # ten records (change for non-experimental implementation) 6334 # However, light responses are not guaranteed with this 6335 # approach, just fewer messages. 6336 limitby = attributes.get('limitby', None) 6337 messages_set = search_result[1][0].split() 6338 # descending order 6339 messages_set.reverse() 6340 if limitby is not None: 6341 # TODO: orderby, asc/desc, limitby from complete message set 6342 messages_set = messages_set[int(limitby[0]):int(limitby[1])] 6343 6344 # keep the requests small for header/flags 6345 if any([(field.name in ["content", "size", 6346 "attachments", "email"]) for 6347 field in fields]): 6348 imap_fields = "(RFC822 FLAGS)" 6349 else: 6350 imap_fields = "(RFC822.HEADER FLAGS)" 6351 6352 if len(messages_set) > 0: 6353 # create fetch results object list 6354 # fetch each remote message and store it in memmory 6355 # (change to multi-fetch command syntax for faster 6356 # transactions) 6357 for uid in messages_set: 6358 # fetch the RFC822 message body 6359 typ, data = self.connection.uid("fetch", uid, imap_fields) 6360 if typ == "OK": 6361 fr = {"message": int(data[0][0].split()[0]), 6362 "uid": long(uid), 6363 "email": email.message_from_string(data[0][1]), 6364 "raw_message": data[0][1]} 6365 fr["multipart"] = fr["email"].is_multipart() 6366 # fetch flags for the message 6367 fr["flags"] = self.driver.ParseFlags(data[1]) 6368 fetch_results.append(fr) 6369 else: 6370 # error retrieving the message body 6371 raise Exception("IMAP error retrieving the body: %s" % data) 6372 else: 6373 raise Exception("IMAP search error: %s" % search_result[1]) 6374 elif isinstance(query, (Expression, basestring)): 6375 raise NotImplementedError() 6376 else: 6377 raise TypeError("Unexpected query type") 6378 6379 imapqry_dict = {} 6380 imapfields_dict = {} 6381 6382 if len(fields) == 1 and isinstance(fields[0], SQLALL): 6383 allfields = True 6384 elif len(fields) == 0: 6385 allfields = True 6386 else: 6387 allfields = False 6388 if allfields: 6389 colnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()] 6390 else: 6391 colnames = ["%s.%s" % (tablename, field.name) for field in fields] 6392 6393 for k in colnames: 6394 imapfields_dict[k] = k 6395 6396 imapqry_list = list() 6397 imapqry_array = list() 6398 for fr in fetch_results: 6399 attachments = [] 6400 content = [] 6401 size = 0 6402 n = int(fr["message"]) 6403 item_dict = dict() 6404 message = fr["email"] 6405 uid = fr["uid"] 6406 charset = self.get_charset(message) 6407 flags = fr["flags"] 6408 raw_message = fr["raw_message"] 6409 # Return messages data mapping static fields 6410 # and fetched results. Mapping should be made 6411 # outside the select function (with auxiliary 6412 # instance methods) 6413 6414 # pending: search flags states trough the email message 6415 # instances for correct output 6416 6417 # preserve subject encoding (ASCII/quoted printable) 6418 6419 if "%s.id" % tablename in colnames: 6420 item_dict["%s.id" % tablename] = n 6421 if "%s.created" % tablename in colnames: 6422 item_dict["%s.created" % tablename] = self.convert_date(message["Date"]) 6423 if "%s.uid" % tablename in colnames: 6424 item_dict["%s.uid" % tablename] = uid 6425 if "%s.sender" % tablename in colnames: 6426 # If there is no encoding found in the message header 6427 # force utf-8 replacing characters (change this to 6428 # module's defaults). Applies to .sender, .to, .cc and .bcc fields 6429 item_dict["%s.sender" % tablename] = message["From"] 6430 if "%s.to" % tablename in colnames: 6431 item_dict["%s.to" % tablename] = message["To"] 6432 if "%s.cc" % tablename in colnames: 6433 if "Cc" in message.keys(): 6434 item_dict["%s.cc" % tablename] = message["Cc"] 6435 else: 6436 item_dict["%s.cc" % tablename] = "" 6437 if "%s.bcc" % tablename in colnames: 6438 if "Bcc" in message.keys(): 6439 item_dict["%s.bcc" % tablename] = message["Bcc"] 6440 else: 6441 item_dict["%s.bcc" % tablename] = "" 6442 if "%s.deleted" % tablename in colnames: 6443 item_dict["%s.deleted" % tablename] = "\\Deleted" in flags 6444 if "%s.draft" % tablename in colnames: 6445 item_dict["%s.draft" % tablename] = "\\Draft" in flags 6446 if "%s.flagged" % tablename in colnames: 6447 item_dict["%s.flagged" % tablename] = "\\Flagged" in flags 6448 if "%s.recent" % tablename in colnames: 6449 item_dict["%s.recent" % tablename] = "\\Recent" in flags 6450 if "%s.seen" % tablename in colnames: 6451 item_dict["%s.seen" % tablename] = "\\Seen" in flags 6452 if "%s.subject" % tablename in colnames: 6453 item_dict["%s.subject" % tablename] = message["Subject"] 6454 if "%s.answered" % tablename in colnames: 6455 item_dict["%s.answered" % tablename] = "\\Answered" in flags 6456 if "%s.mime" % tablename in colnames: 6457 item_dict["%s.mime" % tablename] = message.get_content_type() 6458 if "%s.encoding" % tablename in colnames: 6459 item_dict["%s.encoding" % tablename] = charset 6460 6461 # Here goes the whole RFC822 body as an email instance 6462 # for controller side custom processing 6463 # The message is stored as a raw string 6464 # >> email.message_from_string(raw string) 6465 # returns a Message object for enhanced object processing 6466 if "%s.email" % tablename in colnames: 6467 # WARNING: no encoding performed (raw message) 6468 item_dict["%s.email" % tablename] = raw_message 6469 6470 # Size measure as suggested in a Velocity Reviews post 6471 # by Tim Williams: "how to get size of email attachment" 6472 # Note: len() and server RFC822.SIZE reports doesn't match 6473 # To retrieve the server size for representation would add a new 6474 # fetch transaction to the process 6475 for part in message.walk(): 6476 maintype = part.get_content_maintype() 6477 if ("%s.attachments" % tablename in colnames) or \ 6478 ("%s.content" % tablename in colnames): 6479 payload = part.get_payload(decode=True) 6480 if payload: 6481 filename = part.get_filename() 6482 values = {"mime": part.get_content_type()} 6483 if ((filename or not "text" in maintype) and 6484 ("%s.attachments" % tablename in colnames)): 6485 values.update({"payload": payload, 6486 "filename": filename, 6487 "encoding": part.get_content_charset(), 6488 "disposition": part["Content-Disposition"]}) 6489 attachments.append(values) 6490 elif (("text" in maintype) and 6491 ("%s.content" % tablename in colnames)): 6492 values.update({"text": self.encode_text(payload, 6493 self.get_charset(part))}) 6494 content.append(values) 6495 6496 if "%s.size" % tablename in colnames: 6497 if part is not None: 6498 size += len(str(part)) 6499 item_dict["%s.content" % tablename] = content 6500 item_dict["%s.attachments" % tablename] = attachments 6501 item_dict["%s.size" % tablename] = size 6502 imapqry_list.append(item_dict) 6503 6504 # extra object mapping for the sake of rows object 6505 # creation (sends an array or lists) 6506 for item_dict in imapqry_list: 6507 imapqry_array_item = list() 6508 for fieldname in colnames: 6509 imapqry_array_item.append(item_dict[fieldname]) 6510 imapqry_array.append(imapqry_array_item) 6511 6512 # parse result and return a rows object 6513 colnames = colnames 6514 processor = attributes.get('processor',self.parse) 6515 return processor(imapqry_array, fields, colnames)
6516
6517 - def _insert(self, table, fields):
6518 def add_payload(message, obj): 6519 payload = Message() 6520 payload.set_charset(obj.get("encoding", "utf-8")) 6521 mime = obj.get("mime", None) 6522 if mime: 6523 payload.set_type(mime) 6524 if "text" in obj: 6525 payload.set_payload(obj["text"]) 6526 elif "payload" in obj: 6527 payload.set_payload(obj["payload"]) 6528 if "filename" in obj and obj["filename"]: 6529 payload.add_header("Content-Disposition", 6530 "attachment", filename=obj["filename"]) 6531 message.attach(payload)
6532 6533 mailbox = table.mailbox 6534 d = dict(((k.name, v) for k, v in fields)) 6535 date_time = (d.get("created", datetime.datetime.now())).timetuple() 6536 if len(d) > 0: 6537 message = d.get("email", None) 6538 attachments = d.get("attachments", []) 6539 content = d.get("content", []) 6540 flags = " ".join(["\\%s" % flag.capitalize() for flag in 6541 ("answered", "deleted", "draft", "flagged", 6542 "recent", "seen") if d.get(flag, False)]) 6543 if not message: 6544 from email.message import Message 6545 mime = d.get("mime", None) 6546 charset = d.get("encoding", None) 6547 message = Message() 6548 message["from"] = d.get("sender", "") 6549 message["subject"] = d.get("subject", "") 6550 if mime: 6551 message.set_type(mime) 6552 if charset: 6553 message.set_charset(charset) 6554 for item in ("to", "cc", "bcc"): 6555 value = d.get(item, "") 6556 if isinstance(value, basestring): 6557 message[item] = value 6558 else: 6559 message[item] = ";".join([i for i in 6560 value]) 6561 if (not message.is_multipart() and 6562 (not message.get_content_type().startswith( 6563 "multipart"))): 6564 if isinstance(content, basestring): 6565 message.set_payload(content) 6566 elif len(content) > 0: 6567 message.set_payload(content[0]["text"]) 6568 else: 6569 [add_payload(message, c) for c in content] 6570 [add_payload(message, a) for a in attachments] 6571 message = message.as_string() 6572 return (mailbox, flags, date_time, message) 6573 else: 6574 raise NotImplementedError("IMAP empty insert is not implemented") 6575
6576 - def insert(self, table, fields):
6577 values = self._insert(table, fields) 6578 result, data = self.connection.append(*values) 6579 if result == "OK": 6580 uid = int(re.findall("\d+", str(data))[-1]) 6581 return self.db(table.uid==uid).select(table.id).first().id 6582 else: 6583 raise Exception("IMAP message append failed: %s" % data)
6584
6585 - def _update(self, tablename, query, fields, commit=False):
6586 # TODO: the adapter should implement an .expand method 6587 commands = list() 6588 if use_common_filters(query): 6589 query = self.common_filter(query, [tablename,]) 6590 mark = [] 6591 unmark = [] 6592 if query: 6593 for item in fields: 6594 field = item[0] 6595 name = field.name 6596 value = item[1] 6597 if self.is_flag(name): 6598 flag = self.search_fields[name] 6599 if (value is not None) and (flag != "\\Recent"): 6600 if value: 6601 mark.append(flag) 6602 else: 6603 unmark.append(flag) 6604 result, data = self.connection.select( 6605 self.connection.mailbox_names[tablename]) 6606 string_query = "(%s)" % query 6607 result, data = self.connection.search(None, string_query) 6608 store_list = [item.strip() for item in data[0].split() 6609 if item.strip().isdigit()] 6610 # build commands for marked flags 6611 for number in store_list: 6612 result = None 6613 if len(mark) > 0: 6614 commands.append((number, "+FLAGS", "(%s)" % " ".join(mark))) 6615 if len(unmark) > 0: 6616 commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark))) 6617 return commands
6618
6619 - def update(self, tablename, query, fields):
6620 rowcount = 0 6621 commands = self._update(tablename, query, fields) 6622 for command in commands: 6623 result, data = self.connection.store(*command) 6624 if result == "OK": 6625 rowcount += 1 6626 else: 6627 raise Exception("IMAP storing error: %s" % data) 6628 return rowcount
6629
6630 - def _count(self, query, distinct=None):
6631 raise NotImplementedError()
6632
6633 - def count(self,query,distinct=None):
6634 counter = 0 6635 tablename = self.get_query_mailbox(query) 6636 if query and tablename is not None: 6637 if use_common_filters(query): 6638 query = self.common_filter(query, [tablename,]) 6639 result, data = self.connection.select(self.connection.mailbox_names[tablename]) 6640 string_query = "(%s)" % query 6641 result, data = self.connection.search(None, string_query) 6642 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()] 6643 counter = len(store_list) 6644 return counter
6645
6646 - def delete(self, tablename, query):
6647 counter = 0 6648 if query: 6649 if use_common_filters(query): 6650 query = self.common_filter(query, [tablename,]) 6651 result, data = self.connection.select(self.connection.mailbox_names[tablename]) 6652 string_query = "(%s)" % query 6653 result, data = self.connection.search(None, string_query) 6654 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()] 6655 for number in store_list: 6656 result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)") 6657 if result == "OK": 6658 counter += 1 6659 else: 6660 raise Exception("IMAP store error: %s" % data) 6661 if counter > 0: 6662 result, data = self.connection.expunge() 6663 return counter
6664
6665 - def BELONGS(self, first, second):
6666 result = None 6667 name = self.search_fields[first.name] 6668 if name == "MESSAGE": 6669 values = [str(val) for val in second if str(val).isdigit()] 6670 result = "%s" % ",".join(values).strip() 6671 6672 elif name == "UID": 6673 values = [str(val) for val in second if str(val).isdigit()] 6674 result = "UID %s" % ",".join(values).strip() 6675 6676 else: 6677 raise Exception("Operation not supported") 6678 # result = "(%s %s)" % (self.expand(first), self.expand(second)) 6679 return result
6680
6681 - def CONTAINS(self, first, second, case_sensitive=False):
6682 # silently ignore, only case sensitive 6683 result = None 6684 name = self.search_fields[first.name] 6685 6686 if name in ("FROM", "TO", "SUBJECT", "TEXT"): 6687 result = "%s \"%s\"" % (name, self.expand(second)) 6688 else: 6689 if first.name in ("cc", "bcc"): 6690 result = "%s \"%s\"" % (first.name.upper(), self.expand(second)) 6691 elif first.name == "mime": 6692 result = "HEADER Content-Type \"%s\"" % self.expand(second) 6693 else: 6694 raise Exception("Operation not supported") 6695 return result
6696
6697 - def GT(self, first, second):
6698 result = None 6699 name = self.search_fields[first.name] 6700 if name == "MESSAGE": 6701 last_message = self.get_last_message(first.tablename) 6702 result = "%d:%d" % (int(self.expand(second)) + 1, last_message) 6703 elif name == "UID": 6704 # GT and LT may not return 6705 # expected sets depending on 6706 # the uid format implemented 6707 try: 6708 pedestal, threshold = self.get_uid_bounds(first.tablename) 6709 except TypeError: 6710 e = sys.exc_info()[1] 6711 LOGGER.debug("Error requesting uid bounds: %s", str(e)) 6712 return "" 6713 try: 6714 lower_limit = int(self.expand(second)) + 1 6715 except (ValueError, TypeError): 6716 e = sys.exc_info()[1] 6717 raise Exception("Operation not supported (non integer UID)") 6718 result = "UID %s:%s" % (lower_limit, threshold) 6719 elif name == "DATE": 6720 result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1)) 6721 elif name == "SIZE": 6722 result = "LARGER %s" % self.expand(second) 6723 else: 6724 raise Exception("Operation not supported") 6725 return result
6726
6727 - def GE(self, first, second):
6728 result = None 6729 name = self.search_fields[first.name] 6730 if name == "MESSAGE": 6731 last_message = self.get_last_message(first.tablename) 6732 result = "%s:%s" % (self.expand(second), last_message) 6733 elif name == "UID": 6734 # GT and LT may not return 6735 # expected sets depending on 6736 # the uid format implemented 6737 try: 6738 pedestal, threshold = self.get_uid_bounds(first.tablename) 6739 except TypeError: 6740 e = sys.exc_info()[1] 6741 LOGGER.debug("Error requesting uid bounds: %s", str(e)) 6742 return "" 6743 lower_limit = self.expand(second) 6744 result = "UID %s:%s" % (lower_limit, threshold) 6745 elif name == "DATE": 6746 result = "SINCE %s" % self.convert_date(second) 6747 else: 6748 raise Exception("Operation not supported") 6749 return result
6750
6751 - def LT(self, first, second):
6752 result = None 6753 name = self.search_fields[first.name] 6754 if name == "MESSAGE": 6755 result = "%s:%s" % (1, int(self.expand(second)) - 1) 6756 elif name == "UID": 6757 try: 6758 pedestal, threshold = self.get_uid_bounds(first.tablename) 6759 except TypeError: 6760 e = sys.exc_info()[1] 6761 LOGGER.debug("Error requesting uid bounds: %s", str(e)) 6762 return "" 6763 try: 6764 upper_limit = int(self.expand(second)) - 1 6765 except (ValueError, TypeError): 6766 e = sys.exc_info()[1] 6767 raise Exception("Operation not supported (non integer UID)") 6768 result = "UID %s:%s" % (pedestal, upper_limit) 6769 elif name == "DATE": 6770 result = "BEFORE %s" % self.convert_date(second) 6771 elif name == "SIZE": 6772 result = "SMALLER %s" % self.expand(second) 6773 else: 6774 raise Exception("Operation not supported") 6775 return result
6776
6777 - def LE(self, first, second):
6778 result = None 6779 name = self.search_fields[first.name] 6780 if name == "MESSAGE": 6781 result = "%s:%s" % (1, self.expand(second)) 6782 elif name == "UID": 6783 try: 6784 pedestal, threshold = self.get_uid_bounds(first.tablename) 6785 except TypeError: 6786 e = sys.exc_info()[1] 6787 LOGGER.debug("Error requesting uid bounds: %s", str(e)) 6788 return "" 6789 upper_limit = int(self.expand(second)) 6790 result = "UID %s:%s" % (pedestal, upper_limit) 6791 elif name == "DATE": 6792 result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1)) 6793 else: 6794 raise Exception("Operation not supported") 6795 return result
6796
6797 - def NE(self, first, second=None):
6798 if (second is None) and isinstance(first, Field): 6799 # All records special table query 6800 if first.type == "id": 6801 return self.GE(first, 1) 6802 result = self.NOT(self.EQ(first, second)) 6803 result = result.replace("NOT NOT", "").strip() 6804 return result
6805
6806 - def EQ(self,first,second):
6807 name = self.search_fields[first.name] 6808 result = None 6809 if name is not None: 6810 if name == "MESSAGE": 6811 # query by message sequence number 6812 result = "%s" % self.expand(second) 6813 elif name == "UID": 6814 result = "UID %s" % self.expand(second) 6815 elif name == "DATE": 6816 result = "ON %s" % self.convert_date(second) 6817 6818 elif name in self.flags.values(): 6819 if second: 6820 result = "%s" % (name.upper()[1:]) 6821 else: 6822 result = "NOT %s" % (name.upper()[1:]) 6823 else: 6824 raise Exception("Operation not supported") 6825 else: 6826 raise Exception("Operation not supported") 6827 return result
6828
6829 - def AND(self, first, second):
6830 result = "%s %s" % (self.expand(first), self.expand(second)) 6831 return result
6832
6833 - def OR(self, first, second):
6834 result = "OR %s %s" % (self.expand(first), self.expand(second)) 6835 return "%s" % result.replace("OR OR", "OR")
6836
6837 - def NOT(self, first):
6838 result = "NOT %s" % self.expand(first) 6839 return result
6840 6841 ######################################################################## 6842 # end of adapters 6843 ######################################################################## 6844 6845 ADAPTERS = { 6846 'sqlite': SQLiteAdapter, 6847 'spatialite': SpatiaLiteAdapter, 6848 'sqlite:memory': SQLiteAdapter, 6849 'spatialite:memory': SpatiaLiteAdapter, 6850 'mysql': MySQLAdapter, 6851 'postgres': PostgreSQLAdapter, 6852 'postgres:psycopg2': PostgreSQLAdapter, 6853 'postgres:pg8000': PostgreSQLAdapter, 6854 'postgres2:psycopg2': NewPostgreSQLAdapter, 6855 'postgres2:pg8000': NewPostgreSQLAdapter, 6856 'oracle': OracleAdapter, 6857 'mssql': MSSQLAdapter, 6858 'mssql2': MSSQL2Adapter, 6859 'mssql3': MSSQL3Adapter, 6860 'vertica': VerticaAdapter, 6861 'sybase': SybaseAdapter, 6862 'db2': DB2Adapter, 6863 'teradata': TeradataAdapter, 6864 'informix': InformixAdapter, 6865 'informix-se': InformixSEAdapter, 6866 'firebird': FireBirdAdapter, 6867 'firebird_embedded': FireBirdAdapter, 6868 'ingres': IngresAdapter, 6869 'ingresu': IngresUnicodeAdapter, 6870 'sapdb': SAPDBAdapter, 6871 'cubrid': CubridAdapter, 6872 'jdbc:sqlite': JDBCSQLiteAdapter, 6873 'jdbc:sqlite:memory': JDBCSQLiteAdapter, 6874 'jdbc:postgres': JDBCPostgreSQLAdapter, 6875 'gae': GoogleDatastoreAdapter, # discouraged, for backward compatibility 6876 'google:datastore': GoogleDatastoreAdapter, 6877 'google:sql': GoogleSQLAdapter, 6878 'couchdb': CouchDBAdapter, 6879 'mongodb': MongoDBAdapter, 6880 'imap': IMAPAdapter 6881 }
6882 6883 -def sqlhtml_validators(field):
6884 """ 6885 Field type validation, using web2py's validators mechanism. 6886 6887 makes sure the content of a field is in line with the declared 6888 fieldtype 6889 """ 6890 db = field.db 6891 try: 6892 from gluon import validators 6893 except ImportError: 6894 return [] 6895 field_type, field_length = field.type, field.length 6896 if isinstance(field_type, SQLCustomType): 6897 if hasattr(field_type, 'validator'): 6898 return field_type.validator 6899 else: 6900 field_type = field_type.type 6901 elif not isinstance(field_type,str): 6902 return [] 6903 requires=[] 6904 def ff(r,id): 6905 row=r(id) 6906 if not row: 6907 return id 6908 elif hasattr(r, '_format') and isinstance(r._format,str): 6909 return r._format % row 6910 elif hasattr(r, '_format') and callable(r._format): 6911 return r._format(row) 6912 else: 6913 return id
6914 if field_type in (('string', 'text', 'password')): 6915 requires.append(validators.IS_LENGTH(field_length)) 6916 elif field_type == 'json': 6917 requires.append(validators.IS_EMPTY_OR(validators.IS_JSON(native_json=field.db._adapter.native_json))) 6918 elif field_type == 'double' or field_type == 'float': 6919 requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100)) 6920 elif field_type in ('integer','bigint'): 6921 requires.append(validators.IS_INT_IN_RANGE(-1e100, 1e100)) 6922 elif field_type.startswith('decimal'): 6923 requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10)) 6924 elif field_type == 'date': 6925 requires.append(validators.IS_DATE()) 6926 elif field_type == 'time': 6927 requires.append(validators.IS_TIME()) 6928 elif field_type == 'datetime': 6929 requires.append(validators.IS_DATETIME()) 6930 elif db and field_type.startswith('reference') and \ 6931 field_type.find('.') < 0 and \ 6932 field_type[10:] in db.tables: 6933 referenced = db[field_type[10:]] 6934 def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id) 6935 field.represent = field.represent or repr_ref 6936 if hasattr(referenced, '_format') and referenced._format: 6937 requires = validators.IS_IN_DB(db,referenced._id, 6938 referenced._format) 6939 if field.unique: 6940 requires._and = validators.IS_NOT_IN_DB(db,field) 6941 if field.tablename == field_type[10:]: 6942 return validators.IS_EMPTY_OR(requires) 6943 return requires 6944 elif db and field_type.startswith('list:reference') and \ 6945 field_type.find('.') < 0 and \ 6946 field_type[15:] in db.tables: 6947 referenced = db[field_type[15:]] 6948 def list_ref_repr(ids, row=None, r=referenced, f=ff): 6949 if not ids: 6950 return None 6951 refs = None 6952 db, id = r._db, r._id 6953 if isinstance(db._adapter, GoogleDatastoreAdapter): 6954 def count(values): return db(id.belongs(values)).select(id) 6955 rx = range(0, len(ids), 30) 6956 refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx]) 6957 else: 6958 refs = db(id.belongs(ids)).select(id) 6959 return (refs and ', '.join(f(r,x.id) for x in refs) or '') 6960 field.represent = field.represent or list_ref_repr 6961 if hasattr(referenced, '_format') and referenced._format: 6962 requires = validators.IS_IN_DB(db,referenced._id, 6963 referenced._format,multiple=True) 6964 else: 6965 requires = validators.IS_IN_DB(db,referenced._id, 6966 multiple=True) 6967 if field.unique: 6968 requires._and = validators.IS_NOT_IN_DB(db,field) 6969 if not field.notnull: 6970 requires = validators.IS_EMPTY_OR(requires) 6971 return requires 6972 elif field_type.startswith('list:'): 6973 def repr_list(values,row=None): return', '.join(str(v) for v in (values or [])) 6974 field.represent = field.represent or repr_list 6975 if field.unique: 6976 requires.insert(0,validators.IS_NOT_IN_DB(db,field)) 6977 sff = ['in', 'do', 'da', 'ti', 'de', 'bo'] 6978 if field.notnull and not field_type[:2] in sff: 6979 requires.insert(0, validators.IS_NOT_EMPTY()) 6980 elif not field.notnull and field_type[:2] in sff and requires: 6981 requires[-1] = validators.IS_EMPTY_OR(requires[-1]) 6982 return requires 6983
6984 6985 -def bar_escape(item):
6986 return str(item).replace('|', '||')
6987
6988 -def bar_encode(items):
6989 return '|%s|' % '|'.join(bar_escape(item) for item in items if str(item).strip())
6990
6991 -def bar_decode_integer(value):
6992 if not hasattr(value,'split') and hasattr(value,'read'): 6993 value = value.read() 6994 return [long(x) for x in value.split('|') if x.strip()]
6995
6996 -def bar_decode_string(value):
6997 return [x.replace('||', '|') for x in 6998 REGEX_UNPACK.split(value[1:-1]) if x.strip()]
6999
7000 7001 -class Row(object):
7002 7003 """ 7004 a dictionary that lets you do d['a'] as well as d.a 7005 this is only used to store a Row 7006 """ 7007 7008 __init__ = lambda self,*args,**kwargs: self.__dict__.update(*args,**kwargs) 7009
7010 - def __getitem__(self, k):
7011 key=str(k) 7012 _extra = self.__dict__.get('_extra', None) 7013 if _extra is not None: 7014 v = _extra.get(key, DEFAULT) 7015 if v != DEFAULT: 7016 return v 7017 m = REGEX_TABLE_DOT_FIELD.match(key) 7018 if m: 7019 try: 7020 return ogetattr(self, m.group(1))[m.group(2)] 7021 except (KeyError,AttributeError,TypeError): 7022 key = m.group(2) 7023 try: 7024 return ogetattr(self, key) 7025 except (KeyError,AttributeError,TypeError), ae: 7026 try: 7027 self[key] = ogetattr(self,'__get_lazy_reference__')(key) 7028 return self[key] 7029 except: 7030 raise ae
7031 7032 __setitem__ = lambda self, key, value: setattr(self, str(key), value) 7033 7034 __delitem__ = object.__delattr__ 7035 7036 __copy__ = lambda self: Row(self) 7037 7038 __call__ = __getitem__ 7039 7040
7041 - def get(self, key, default=None):
7042 try: 7043 return self.__getitem__(key) 7044 except(KeyError, AttributeError, TypeError): 7045 return self.__dict__.get(key,default)
7046 7047 has_key = __contains__ = lambda self, key: key in self.__dict__ 7048 7049 __nonzero__ = lambda self: len(self.__dict__)>0 7050 7051 update = lambda self, *args, **kwargs: self.__dict__.update(*args, **kwargs) 7052 7053 keys = lambda self: self.__dict__.keys() 7054 7055 items = lambda self: self.__dict__.items() 7056 7057 values = lambda self: self.__dict__.values() 7058 7059 __iter__ = lambda self: self.__dict__.__iter__() 7060 7061 iteritems = lambda self: self.__dict__.iteritems() 7062 7063 __str__ = __repr__ = lambda self: '<Row %s>' % self.as_dict() 7064 7065 __int__ = lambda self: object.__getattribute__(self,'id') 7066 7067 __long__ = lambda self: long(object.__getattribute__(self,'id')) 7068 7069 __getattr__ = __getitem__ 7070 7071 # def __getattribute__(self, key): 7072 # try: 7073 # return object.__getattribute__(self, key) 7074 # except AttributeError, ae: 7075 # try: 7076 # return self.__get_lazy_reference__(key) 7077 # except: 7078 # raise ae 7079
7080 - def __eq__(self,other):
7081 try: 7082 return self.as_dict() == other.as_dict() 7083 except AttributeError: 7084 return False
7085
7086 - def __ne__(self,other):
7087 return not (self == other)
7088
7089 - def __copy__(self):
7090 return Row(dict(self))
7091
7092 - def as_dict(self, datetime_to_str=False, custom_types=None):
7093 SERIALIZABLE_TYPES = [str, unicode, int, long, float, bool, list, dict] 7094 if isinstance(custom_types,(list,tuple,set)): 7095 SERIALIZABLE_TYPES += custom_types 7096 elif custom_types: 7097 SERIALIZABLE_TYPES.append(custom_types) 7098 d = dict(self) 7099 for k in copy.copy(d.keys()): 7100 v=d[k] 7101 if d[k] is None: 7102 continue 7103 elif isinstance(v,Row): 7104 d[k]=v.as_dict() 7105 elif isinstance(v,Reference): 7106 d[k]=long(v) 7107 elif isinstance(v,decimal.Decimal): 7108 d[k]=float(v) 7109 elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)): 7110 if datetime_to_str: 7111 d[k] = v.isoformat().replace('T',' ')[:19] 7112 elif not isinstance(v,tuple(SERIALIZABLE_TYPES)): 7113 del d[k] 7114 return d
7115
7116 - def as_xml(self, row_name="row", colnames=None, indent=' '):
7117 def f(row,field,indent=' '): 7118 if isinstance(row,Row): 7119 spc = indent+' \n' 7120 items = [f(row[x],x,indent+' ') for x in row] 7121 return '%s<%s>\n%s\n%s</%s>' % ( 7122 indent, 7123 field, 7124 spc.join(item for item in items if item), 7125 indent, 7126 field) 7127 elif not callable(row): 7128 if REGEX_ALPHANUMERIC.match(field): 7129 return '%s<%s>%s</%s>' % (indent,field,row,field) 7130 else: 7131 return '%s<extra name="%s">%s</extra>' % \ 7132 (indent,field,row) 7133 else: 7134 return None
7135 return f(self, row_name, indent=indent)
7136
7137 - def as_json(self, mode="object", default=None, colnames=None, 7138 serialize=True, **kwargs):
7139 """ 7140 serializes the row to a JSON object 7141 kwargs are passed to .as_dict method 7142 only "object" mode supported 7143 7144 serialize = False used by Rows.as_json 7145 TODO: return array mode with query column order 7146 7147 mode and colnames are not implemented 7148 """ 7149 7150 item = self.as_dict(**kwargs) 7151 if serialize: 7152 if have_serializers: 7153 return serializers.json(item, 7154 default=default or 7155 serializers.custom_json) 7156 elif simplejson: 7157 return simplejson.dumps(item) 7158 else: 7159 raise RuntimeError("missing simplejson") 7160 else: 7161 return item
7162
7163 7164 ################################################################################ 7165 # Everything below should be independent of the specifics of the database 7166 # and should work for RDBMs and some NoSQL databases 7167 ################################################################################ 7168 7169 -class SQLCallableList(list):
7170 - def __call__(self):
7171 return copy.copy(self)
7172
7173 -def smart_query(fields,text):
7174 if not isinstance(fields,(list,tuple)): 7175 fields = [fields] 7176 new_fields = [] 7177 for field in fields: 7178 if isinstance(field,Field): 7179 new_fields.append(field) 7180 elif isinstance(field,Table): 7181 for ofield in field: 7182 new_fields.append(ofield) 7183 else: 7184 raise RuntimeError("fields must be a list of fields") 7185 fields = new_fields 7186 field_map = {} 7187 for field in fields: 7188 n = field.name.lower() 7189 if not n in field_map: 7190 field_map[n] = field 7191 n = str(field).lower() 7192 if not n in field_map: 7193 field_map[n] = field 7194 constants = {} 7195 i = 0 7196 while True: 7197 m = REGEX_CONST_STRING.search(text) 7198 if not m: break 7199 text = text[:m.start()]+('#%i' % i)+text[m.end():] 7200 constants[str(i)] = m.group()[1:-1] 7201 i+=1 7202 text = re.sub('\s+',' ',text).lower() 7203 for a,b in [('&','and'), 7204 ('|','or'), 7205 ('~','not'), 7206 ('==','='), 7207 ('<','<'), 7208 ('>','>'), 7209 ('<=','<='), 7210 ('>=','>='), 7211 ('<>','!='), 7212 ('=<','<='), 7213 ('=>','>='), 7214 ('=','='), 7215 (' less or equal than ','<='), 7216 (' greater or equal than ','>='), 7217 (' equal or less than ','<='), 7218 (' equal or greater than ','>='), 7219 (' less or equal ','<='), 7220 (' greater or equal ','>='), 7221 (' equal or less ','<='), 7222 (' equal or greater ','>='), 7223 (' not equal to ','!='), 7224 (' not equal ','!='), 7225 (' equal to ','='), 7226 (' equal ','='), 7227 (' equals ','='), 7228 (' less than ','<'), 7229 (' greater than ','>'), 7230 (' starts with ','startswith'), 7231 (' ends with ','endswith'), 7232 (' not in ' , 'notbelongs'), 7233 (' in ' , 'belongs'), 7234 (' is ','=')]: 7235 if a[0]==' ': 7236 text = text.replace(' is'+a,' %s ' % b) 7237 text = text.replace(a,' %s ' % b) 7238 text = re.sub('\s+',' ',text).lower() 7239 text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text) 7240 query = field = neg = op = logic = None 7241 for item in text.split(): 7242 if field is None: 7243 if item == 'not': 7244 neg = True 7245 elif not neg and not logic and item in ('and','or'): 7246 logic = item 7247 elif item in field_map: 7248 field = field_map[item] 7249 else: 7250 raise RuntimeError("Invalid syntax") 7251 elif not field is None and op is None: 7252 op = item 7253 elif not op is None: 7254 if item.startswith('#'): 7255 if not item[1:] in constants: 7256 raise RuntimeError("Invalid syntax") 7257 value = constants[item[1:]] 7258 else: 7259 value = item 7260 if field.type in ('text', 'string', 'json'): 7261 if op == '=': op = 'like' 7262 if op == '=': new_query = field==value 7263 elif op == '<': new_query = field<value 7264 elif op == '>': new_query = field>value 7265 elif op == '<=': new_query = field<=value 7266 elif op == '>=': new_query = field>=value 7267 elif op == '!=': new_query = field!=value 7268 elif op == 'belongs': new_query = field.belongs(value.split(',')) 7269 elif op == 'notbelongs': new_query = ~field.belongs(value.split(',')) 7270 elif field.type in ('text', 'string', 'json'): 7271 if op == 'contains': new_query = field.contains(value) 7272 elif op == 'like': new_query = field.like(value) 7273 elif op == 'startswith': new_query = field.startswith(value) 7274 elif op == 'endswith': new_query = field.endswith(value) 7275 else: raise RuntimeError("Invalid operation") 7276 elif field._db._adapter.dbengine=='google:datastore' and \ 7277 field.type in ('list:integer', 'list:string', 'list:reference'): 7278 if op == 'contains': new_query = field.contains(value) 7279 else: raise RuntimeError("Invalid operation") 7280 else: raise RuntimeError("Invalid operation") 7281 if neg: new_query = ~new_query 7282 if query is None: 7283 query = new_query 7284 elif logic == 'and': 7285 query &= new_query 7286 elif logic == 'or': 7287 query |= new_query 7288 field = op = neg = logic = None 7289 return query
7290
7291 -class DAL(object):
7292 7293 """ 7294 an instance of this class represents a database connection 7295 7296 Example:: 7297 7298 db = DAL('sqlite://test.db') 7299 7300 or 7301 7302 db = DAL(**{"uri": ..., "tables": [...]...}) # experimental 7303 7304 db.define_table('tablename', Field('fieldname1'), 7305 Field('fieldname2')) 7306 """ 7307
7308 - def __new__(cls, uri='sqlite://dummy.db', *args, **kwargs):
7309 if not hasattr(THREAD_LOCAL,'db_instances'): 7310 THREAD_LOCAL.db_instances = {} 7311 if not hasattr(THREAD_LOCAL,'db_instances_zombie'): 7312 THREAD_LOCAL.db_instances_zombie = {} 7313 if uri == '<zombie>': 7314 db_uid = kwargs['db_uid'] # a zombie must have a db_uid! 7315 if db_uid in THREAD_LOCAL.db_instances: 7316 db_group = THREAD_LOCAL.db_instances[db_uid] 7317 db = db_group[-1] 7318 elif db_uid in THREAD_LOCAL.db_instances_zombie: 7319 db = THREAD_LOCAL.db_instances_zombie[db_uid] 7320 else: 7321 db = super(DAL, cls).__new__(cls) 7322 THREAD_LOCAL.db_instances_zombie[db_uid] = db 7323 else: 7324 db_uid = kwargs.get('db_uid',hashlib_md5(repr(uri)).hexdigest()) 7325 if db_uid in THREAD_LOCAL.db_instances_zombie: 7326 db = THREAD_LOCAL.db_instances_zombie[db_uid] 7327 del THREAD_LOCAL.db_instances_zombie[db_uid] 7328 else: 7329 db = super(DAL, cls).__new__(cls) 7330 db_group = THREAD_LOCAL.db_instances.get(db_uid,[]) 7331 db_group.append(db) 7332 THREAD_LOCAL.db_instances[db_uid] = db_group 7333 db._db_uid = db_uid 7334 return db
7335 7336 @staticmethod
7337 - def set_folder(folder):
7338 """ 7339 # ## this allows gluon to set a folder for this thread 7340 # ## <<<<<<<<< Should go away as new DAL replaces old sql.py 7341 """ 7342 BaseAdapter.set_folder(folder)
7343 7344 @staticmethod
7345 - def get_instances():
7346 """ 7347 Returns a dictionary with uri as key with timings and defined tables 7348 {'sqlite://storage.sqlite': { 7349 'dbstats': [(select auth_user.email from auth_user, 0.02009)], 7350 'dbtables': { 7351 'defined': ['auth_cas', 'auth_event', 'auth_group', 7352 'auth_membership', 'auth_permission', 'auth_user'], 7353 'lazy': '[]' 7354 } 7355 } 7356 } 7357 """ 7358 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items() 7359 infos = {} 7360 for db_uid, db_group in dbs: 7361 for db in db_group: 7362 if not db._uri: 7363 continue 7364 k = hide_password(db._uri) 7365 infos[k] = dict(dbstats = [(row[0], row[1]) for row in db._timings], 7366 dbtables = {'defined': 7367 sorted(list(set(db.tables) - 7368 set(db._LAZY_TABLES.keys()))), 7369 'lazy': sorted(db._LAZY_TABLES.keys())} 7370 ) 7371 return infos
7372 7373 @staticmethod
7374 - def distributed_transaction_begin(*instances):
7375 if not instances: 7376 return 7377 thread_key = '%s.%s' % (socket.gethostname(), threading.currentThread()) 7378 keys = ['%s.%i' % (thread_key, i) for (i,db) in instances] 7379 instances = enumerate(instances) 7380 for (i, db) in instances: 7381 if not db._adapter.support_distributed_transaction(): 7382 raise SyntaxError( 7383 'distributed transaction not suported by %s' % db._dbname) 7384 for (i, db) in instances: 7385 db._adapter.distributed_transaction_begin(keys[i])
7386 7387 @staticmethod
7388 - def distributed_transaction_commit(*instances):
7389 if not instances: 7390 return 7391 instances = enumerate(instances) 7392 thread_key = '%s.%s' % (socket.gethostname(), threading.currentThread()) 7393 keys = ['%s.%i' % (thread_key, i) for (i,db) in instances] 7394 for (i, db) in instances: 7395 if not db._adapter.support_distributed_transaction(): 7396 raise SyntaxError( 7397 'distributed transaction not suported by %s' % db._dbanme) 7398 try: 7399 for (i, db) in instances: 7400 db._adapter.prepare(keys[i]) 7401 except: 7402 for (i, db) in instances: 7403 db._adapter.rollback_prepared(keys[i]) 7404 raise RuntimeError('failure to commit distributed transaction') 7405 else: 7406 for (i, db) in instances: 7407 db._adapter.commit_prepared(keys[i]) 7408 return
7409
7410 - def __init__(self, uri=DEFAULT_URI, 7411 pool_size=0, folder=None, 7412 db_codec='UTF-8', check_reserved=None, 7413 migrate=True, fake_migrate=False, 7414 migrate_enabled=True, fake_migrate_all=False, 7415 decode_credentials=False, driver_args=None, 7416 adapter_args=None, attempts=5, auto_import=False, 7417 bigint_id=False, debug=False, lazy_tables=False, 7418 db_uid=None, do_connect=True, 7419 after_connection=None, tables=None):
7420 """ 7421 Creates a new Database Abstraction Layer instance. 7422 7423 Keyword arguments: 7424 7425 :uri: string that contains information for connecting to a database. 7426 (default: 'sqlite://dummy.db') 7427 7428 experimental: you can specify a dictionary as uri 7429 parameter i.e. with 7430 db = DAL({"uri": "sqlite://storage.sqlite", 7431 "tables": {...}, ...}) 7432 7433 for an example of dict input you can check the output 7434 of the scaffolding db model with 7435 7436 db.as_dict() 7437 7438 Note that for compatibility with Python older than 7439 version 2.6.5 you should cast your dict input keys 7440 to str due to a syntax limitation on kwarg names. 7441 for proper DAL dictionary input you can use one of: 7442 7443 obj = serializers.cast_keys(dict, [encoding="utf-8"]) 7444 7445 or else (for parsing json input) 7446 7447 obj = serializers.loads_json(data, unicode_keys=False) 7448 7449 :pool_size: How many open connections to make to the database object. 7450 :folder: where .table files will be created. 7451 automatically set within web2py 7452 use an explicit path when using DAL outside web2py 7453 :db_codec: string encoding of the database (default: 'UTF-8') 7454 :check_reserved: list of adapters to check tablenames and column names 7455 against sql/nosql reserved keywords. (Default None) 7456 7457 * 'common' List of sql keywords that are common to all database types 7458 such as "SELECT, INSERT". (recommended) 7459 * 'all' Checks against all known SQL keywords. (not recommended) 7460 <adaptername> Checks against the specific adapters list of keywords 7461 (recommended) 7462 * '<adaptername>_nonreserved' Checks against the specific adapters 7463 list of nonreserved keywords. (if available) 7464 :migrate (defaults to True) sets default migrate behavior for all tables 7465 :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables 7466 :migrate_enabled (defaults to True). If set to False disables ALL migrations 7467 :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables 7468 :attempts (defaults to 5). Number of times to attempt connecting 7469 :auto_import (defaults to False). If set, import automatically table definitions from the 7470 databases folder 7471 :bigint_id (defaults to False): If set, turn on bigint instead of int for id fields 7472 :lazy_tables (defaults to False): delay table definition until table access 7473 :after_connection (defaults to None): a callable that will be execute after the connection 7474 """ 7475 if uri == '<zombie>' and db_uid is not None: return 7476 if not decode_credentials: 7477 credential_decoder = lambda cred: cred 7478 else: 7479 credential_decoder = lambda cred: urllib.unquote(cred) 7480 self._folder = folder 7481 if folder: 7482 self.set_folder(folder) 7483 self._uri = uri 7484 self._pool_size = pool_size 7485 self._db_codec = db_codec 7486 self._lastsql = '' 7487 self._timings = [] 7488 self._pending_references = {} 7489 self._request_tenant = 'request_tenant' 7490 self._common_fields = [] 7491 self._referee_name = '%(table)s' 7492 self._bigint_id = bigint_id 7493 self._debug = debug 7494 self._migrated = [] 7495 self._LAZY_TABLES = {} 7496 self._lazy_tables = lazy_tables 7497 self._tables = SQLCallableList() 7498 self._driver_args = driver_args 7499 self._adapter_args = adapter_args 7500 self._check_reserved = check_reserved 7501 self._decode_credentials = decode_credentials 7502 self._attempts = attempts 7503 self._do_connect = do_connect 7504 7505 if not str(attempts).isdigit() or attempts < 0: 7506 attempts = 5 7507 if uri: 7508 uris = isinstance(uri,(list,tuple)) and uri or [uri] 7509 error = '' 7510 connected = False 7511 for k in range(attempts): 7512 for uri in uris: 7513 try: 7514 if is_jdbc and not uri.startswith('jdbc:'): 7515 uri = 'jdbc:'+uri 7516 self._dbname = REGEX_DBNAME.match(uri).group() 7517 if not self._dbname in ADAPTERS: 7518 raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname) 7519 # notice that driver args or {} else driver_args 7520 # defaults to {} global, not correct 7521 kwargs = dict(db=self,uri=uri, 7522 pool_size=pool_size, 7523 folder=folder, 7524 db_codec=db_codec, 7525 credential_decoder=credential_decoder, 7526 driver_args=driver_args or {}, 7527 adapter_args=adapter_args or {}, 7528 do_connect=do_connect, 7529 after_connection=after_connection) 7530 self._adapter = ADAPTERS[self._dbname](**kwargs) 7531 types = ADAPTERS[self._dbname].types 7532 # copy so multiple DAL() possible 7533 self._adapter.types = copy.copy(types) 7534 self._adapter.build_parsemap() 7535 if bigint_id: 7536 if 'big-id' in types and 'reference' in types: 7537 self._adapter.types['id'] = types['big-id'] 7538 self._adapter.types['reference'] = types['big-reference'] 7539 connected = True 7540 break 7541 except SyntaxError: 7542 raise 7543 except Exception: 7544 tb = traceback.format_exc() 7545 sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb)) 7546 if connected: 7547 break 7548 else: 7549 time.sleep(1) 7550 if not connected: 7551 raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb)) 7552 else: 7553 self._adapter = BaseAdapter(db=self,pool_size=0, 7554 uri='None',folder=folder, 7555 db_codec=db_codec, after_connection=after_connection) 7556 migrate = fake_migrate = False 7557 adapter = self._adapter 7558 self._uri_hash = hashlib_md5(adapter.uri).hexdigest() 7559 self.check_reserved = check_reserved 7560 if self.check_reserved: 7561 from reserved_sql_keywords import ADAPTERS as RSK 7562 self.RSK = RSK 7563 self._migrate = migrate 7564 self._fake_migrate = fake_migrate 7565 self._migrate_enabled = migrate_enabled 7566 self._fake_migrate_all = fake_migrate_all 7567 if auto_import or tables: 7568 self.import_table_definitions(adapter.folder, 7569 tables=tables)
7570 7571 @property
7572 - def tables(self):
7573 return self._tables
7574
7575 - def import_table_definitions(self, path, migrate=False, 7576 fake_migrate=False, tables=None):
7577 pattern = pjoin(path,self._uri_hash+'_*.table') 7578 if tables: 7579 for table in tables: 7580 self.define_table(**table) 7581 else: 7582 for filename in glob.glob(pattern): 7583 tfile = self._adapter.file_open(filename, 'r') 7584 try: 7585 sql_fields = pickle.load(tfile) 7586 name = filename[len(pattern)-7:-6] 7587 mf = [(value['sortable'], 7588 Field(key, 7589 type=value['type'], 7590 length=value.get('length',None), 7591 notnull=value.get('notnull',False), 7592 unique=value.get('unique',False))) \ 7593 for key, value in sql_fields.iteritems()] 7594 mf.sort(lambda a,b: cmp(a[0],b[0])) 7595 self.define_table(name,*[item[1] for item in mf], 7596 **dict(migrate=migrate, 7597 fake_migrate=fake_migrate)) 7598 finally: 7599 self._adapter.file_close(tfile)
7600
7601 - def check_reserved_keyword(self, name):
7602 """ 7603 Validates ``name`` against SQL keywords 7604 Uses self.check_reserve which is a list of 7605 operators to use. 7606 self.check_reserved 7607 ['common', 'postgres', 'mysql'] 7608 self.check_reserved 7609 ['all'] 7610 """ 7611 for backend in self.check_reserved: 7612 if name.upper() in self.RSK[backend]: 7613 raise SyntaxError( 7614 'invalid table/column name "%s" is a "%s" reserved SQL/NOSQL keyword' % (name, backend.upper()))
7615
7616 - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
7617 """ 7618 EXAMPLE: 7619 7620 db.define_table('person',Field('name'),Field('info')) 7621 db.define_table('pet',Field('ownedby',db.person),Field('name'),Field('info')) 7622 7623 @request.restful() 7624 def index(): 7625 def GET(*args,**vars): 7626 patterns = [ 7627 "/friends[person]", 7628 "/{person.name}/:field", 7629 "/{person.name}/pets[pet.ownedby]", 7630 "/{person.name}/pets[pet.ownedby]/{pet.name}", 7631 "/{person.name}/pets[pet.ownedby]/{pet.name}/:field", 7632 ("/dogs[pet]", db.pet.info=='dog'), 7633 ("/dogs[pet]/{pet.name.startswith}", db.pet.info=='dog'), 7634 ] 7635 parser = db.parse_as_rest(patterns,args,vars) 7636 if parser.status == 200: 7637 return dict(content=parser.response) 7638 else: 7639 raise HTTP(parser.status,parser.error) 7640 7641 def POST(table_name,**vars): 7642 if table_name == 'person': 7643 return db.person.validate_and_insert(**vars) 7644 elif table_name == 'pet': 7645 return db.pet.validate_and_insert(**vars) 7646 else: 7647 raise HTTP(400) 7648 return locals() 7649 """ 7650 7651 db = self 7652 re1 = REGEX_SEARCH_PATTERN 7653 re2 = REGEX_SQUARE_BRACKETS 7654 7655 def auto_table(table,base='',depth=0): 7656 patterns = [] 7657 for field in db[table].fields: 7658 if base: 7659 tag = '%s/%s' % (base,field.replace('_','-')) 7660 else: 7661 tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-')) 7662 f = db[table][field] 7663 if not f.readable: continue 7664 if f.type=='id' or 'slug' in field or f.type.startswith('reference'): 7665 tag += '/{%s.%s}' % (table,field) 7666 patterns.append(tag) 7667 patterns.append(tag+'/:field') 7668 elif f.type.startswith('boolean'): 7669 tag += '/{%s.%s}' % (table,field) 7670 patterns.append(tag) 7671 patterns.append(tag+'/:field') 7672 elif f.type in ('float','double','integer','bigint'): 7673 tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field) 7674 patterns.append(tag) 7675 patterns.append(tag+'/:field') 7676 elif f.type.startswith('list:'): 7677 tag += '/{%s.%s.contains}' % (table,field) 7678 patterns.append(tag) 7679 patterns.append(tag+'/:field') 7680 elif f.type in ('date','datetime'): 7681 tag+= '/{%s.%s.year}' % (table,field) 7682 patterns.append(tag) 7683 patterns.append(tag+'/:field') 7684 tag+='/{%s.%s.month}' % (table,field) 7685 patterns.append(tag) 7686 patterns.append(tag+'/:field') 7687 tag+='/{%s.%s.day}' % (table,field) 7688 patterns.append(tag) 7689 patterns.append(tag+'/:field') 7690 if f.type in ('datetime','time'): 7691 tag+= '/{%s.%s.hour}' % (table,field) 7692 patterns.append(tag) 7693 patterns.append(tag+'/:field') 7694 tag+='/{%s.%s.minute}' % (table,field) 7695 patterns.append(tag) 7696 patterns.append(tag+'/:field') 7697 tag+='/{%s.%s.second}' % (table,field) 7698 patterns.append(tag) 7699 patterns.append(tag+'/:field') 7700 if depth>0: 7701 for f in db[table]._referenced_by: 7702 tag+='/%s[%s.%s]' % (table,f.tablename,f.name) 7703 patterns.append(tag) 7704 patterns += auto_table(table,base=tag,depth=depth-1) 7705 return patterns
7706 7707 if patterns == 'auto': 7708 patterns=[] 7709 for table in db.tables: 7710 if not table.startswith('auth_'): 7711 patterns.append('/%s[%s]' % (table,table)) 7712 patterns += auto_table(table,base='',depth=1) 7713 else: 7714 i = 0 7715 while i<len(patterns): 7716 pattern = patterns[i] 7717 if not isinstance(pattern,str): 7718 pattern = pattern[0] 7719 tokens = pattern.split('/') 7720 if tokens[-1].startswith(':auto') and re2.match(tokens[-1]): 7721 new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1], 7722 '/'.join(tokens[:-1])) 7723 patterns = patterns[:i]+new_patterns+patterns[i+1:] 7724 i += len(new_patterns) 7725 else: 7726 i += 1 7727 if '/'.join(args) == 'patterns': 7728 return Row({'status':200,'pattern':'list', 7729 'error':None,'response':patterns}) 7730 for pattern in patterns: 7731 basequery, exposedfields = None, [] 7732 if isinstance(pattern,tuple): 7733 if len(pattern)==2: 7734 pattern, basequery = pattern 7735 elif len(pattern)>2: 7736 pattern, basequery, exposedfields = pattern[0:3] 7737 otable=table=None 7738 if not isinstance(queries,dict): 7739 dbset=db(queries) 7740 if basequery is not None: 7741 dbset = dbset(basequery) 7742 i=0 7743 tags = pattern[1:].split('/') 7744 if len(tags)!=len(args): 7745 continue 7746 for tag in tags: 7747 if re1.match(tag): 7748 # print 're1:'+tag 7749 tokens = tag[1:-1].split('.') 7750 table, field = tokens[0], tokens[1] 7751 if not otable or table == otable: 7752 if len(tokens)==2 or tokens[2]=='eq': 7753 query = db[table][field]==args[i] 7754 elif tokens[2]=='ne': 7755 query = db[table][field]!=args[i] 7756 elif tokens[2]=='lt': 7757 query = db[table][field]<args[i] 7758 elif tokens[2]=='gt': 7759 query = db[table][field]>args[i] 7760 elif tokens[2]=='ge': 7761 query = db[table][field]>=args[i] 7762 elif tokens[2]=='le': 7763 query = db[table][field]<=args[i] 7764 elif tokens[2]=='year': 7765 query = db[table][field].year()==args[i] 7766 elif tokens[2]=='month': 7767 query = db[table][field].month()==args[i] 7768 elif tokens[2]=='day': 7769 query = db[table][field].day()==args[i] 7770 elif tokens[2]=='hour': 7771 query = db[table][field].hour()==args[i] 7772 elif tokens[2]=='minute': 7773 query = db[table][field].minutes()==args[i] 7774 elif tokens[2]=='second': 7775 query = db[table][field].seconds()==args[i] 7776 elif tokens[2]=='startswith': 7777 query = db[table][field].startswith(args[i]) 7778 elif tokens[2]=='contains': 7779 query = db[table][field].contains(args[i]) 7780 else: 7781 raise RuntimeError("invalid pattern: %s" % pattern) 7782 if len(tokens)==4 and tokens[3]=='not': 7783 query = ~query 7784 elif len(tokens)>=4: 7785 raise RuntimeError("invalid pattern: %s" % pattern) 7786 if not otable and isinstance(queries,dict): 7787 dbset = db(queries[table]) 7788 if basequery is not None: 7789 dbset = dbset(basequery) 7790 dbset=dbset(query) 7791 else: 7792 raise RuntimeError("missing relation in pattern: %s" % pattern) 7793 elif re2.match(tag) and args[i]==tag[:tag.find('[')]: 7794 ref = tag[tag.find('[')+1:-1] 7795 if '.' in ref and otable: 7796 table,field = ref.split('.') 7797 selfld = '_id' 7798 if db[table][field].type.startswith('reference '): 7799 refs = [ x.name for x in db[otable] if x.type == db[table][field].type ] 7800 else: 7801 refs = [ x.name for x in db[table]._referenced_by if x.tablename==otable ] 7802 if refs: 7803 selfld = refs[0] 7804 if nested_select: 7805 try: 7806 dbset=db(db[table][field].belongs(dbset._select(db[otable][selfld]))) 7807 except ValueError: 7808 return Row({'status':400,'pattern':pattern, 7809 'error':'invalid path','response':None}) 7810 else: 7811 items = [item.id for item in dbset.select(db[otable][selfld])] 7812 dbset=db(db[table][field].belongs(items)) 7813 else: 7814 table = ref 7815 if not otable and isinstance(queries,dict): 7816 dbset = db(queries[table]) 7817 dbset=dbset(db[table]) 7818 elif tag==':field' and table: 7819 # print 're3:'+tag 7820 field = args[i] 7821 if not field in db[table]: break 7822 # hand-built patterns should respect .readable=False as well 7823 if not db[table][field].readable: 7824 return Row({'status':418,'pattern':pattern, 7825 'error':'I\'m a teapot','response':None}) 7826 try: 7827 distinct = vars.get('distinct', False) == 'True' 7828 offset = long(vars.get('offset',None) or 0) 7829 limits = (offset,long(vars.get('limit',None) or 1000)+offset) 7830 except ValueError: 7831 return Row({'status':400,'error':'invalid limits','response':None}) 7832 items = dbset.select(db[table][field], distinct=distinct, limitby=limits) 7833 if items: 7834 return Row({'status':200,'response':items, 7835 'pattern':pattern}) 7836 else: 7837 return Row({'status':404,'pattern':pattern, 7838 'error':'no record found','response':None}) 7839 elif tag != args[i]: 7840 break 7841 otable = table 7842 i += 1 7843 if i==len(tags) and table: 7844 ofields = vars.get('order',db[table]._id.name).split('|') 7845 try: 7846 orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields] 7847 except (KeyError, AttributeError): 7848 return Row({'status':400,'error':'invalid orderby','response':None}) 7849 if exposedfields: 7850 fields = [field for field in db[table] if str(field).split('.')[-1] in exposedfields and field.readable] 7851 else: 7852 fields = [field for field in db[table] if field.readable] 7853 count = dbset.count() 7854 try: 7855 offset = long(vars.get('offset',None) or 0) 7856 limits = (offset,long(vars.get('limit',None) or 1000)+offset) 7857 except ValueError: 7858 return Row({'status':400,'error':'invalid limits','response':None}) 7859 if count > limits[1]-limits[0]: 7860 return Row({'status':400,'error':'too many records','response':None}) 7861 try: 7862 response = dbset.select(limitby=limits,orderby=orderby,*fields) 7863 except ValueError: 7864 return Row({'status':400,'pattern':pattern, 7865 'error':'invalid path','response':None}) 7866 return Row({'status':200,'response':response, 7867 'pattern':pattern,'count':count}) 7868 return Row({'status':400,'error':'no matching pattern','response':None})
7869
7870 - def define_table( 7871 self, 7872 tablename, 7873 *fields, 7874 **args 7875 ):
7876 if not fields and 'fields' in args: 7877 fields = args.get('fields',()) 7878 if not isinstance(tablename, str): 7879 if isinstance(tablename, unicode): 7880 try: 7881 tablename = str(tablename) 7882 except UnicodeEncodeError: 7883 raise SyntaxError("invalid unicode table name") 7884 else: 7885 raise SyntaxError("missing table name") 7886 elif hasattr(self,tablename) or tablename in self.tables: 7887 if not args.get('redefine',False): 7888 raise SyntaxError('table already defined: %s' % tablename) 7889 elif tablename.startswith('_') or hasattr(self,tablename) or \ 7890 REGEX_PYTHON_KEYWORDS.match(tablename): 7891 raise SyntaxError('invalid table name: %s' % tablename) 7892 elif self.check_reserved: 7893 self.check_reserved_keyword(tablename) 7894 else: 7895 invalid_args = set(args)-TABLE_ARGS 7896 if invalid_args: 7897 raise SyntaxError('invalid table "%s" attributes: %s' \ 7898 % (tablename,invalid_args)) 7899 if self._lazy_tables and not tablename in self._LAZY_TABLES: 7900 self._LAZY_TABLES[tablename] = (tablename,fields,args) 7901 table = None 7902 else: 7903 table = self.lazy_define_table(tablename,*fields,**args) 7904 if not tablename in self.tables: 7905 self.tables.append(tablename) 7906 return table
7907
7908 - def lazy_define_table( 7909 self, 7910 tablename, 7911 *fields, 7912 **args 7913 ):
7914 args_get = args.get 7915 common_fields = self._common_fields 7916 if common_fields: 7917 fields = list(fields) + list(common_fields) 7918 7919 table_class = args_get('table_class',Table) 7920 table = table_class(self, tablename, *fields, **args) 7921 table._actual = True 7922 self[tablename] = table 7923 # must follow above line to handle self references 7924 table._create_references() 7925 for field in table: 7926 if field.requires == DEFAULT: 7927 field.requires = sqlhtml_validators(field) 7928 7929 migrate = self._migrate_enabled and args_get('migrate',self._migrate) 7930 if migrate and not self._uri in (None,'None') \ 7931 or self._adapter.dbengine=='google:datastore': 7932 fake_migrate = self._fake_migrate_all or \ 7933 args_get('fake_migrate',self._fake_migrate) 7934 polymodel = args_get('polymodel',None) 7935 try: 7936 GLOBAL_LOCKER.acquire() 7937 self._lastsql = self._adapter.create_table( 7938 table,migrate=migrate, 7939 fake_migrate=fake_migrate, 7940 polymodel=polymodel) 7941 finally: 7942 GLOBAL_LOCKER.release() 7943 else: 7944 table._dbt = None 7945 on_define = args_get('on_define',None) 7946 if on_define: on_define(table) 7947 return table
7948
7949 - def as_dict(self, flat=False, sanitize=True):
7950 db_uid = uri = None 7951 if not sanitize: 7952 uri, db_uid = (self._uri, self._db_uid) 7953 db_as_dict = dict(tables=[], uri=uri, db_uid=db_uid, 7954 **dict([(k, getattr(self, "_" + k, None)) 7955 for k in 'pool_size','folder','db_codec', 7956 'check_reserved','migrate','fake_migrate', 7957 'migrate_enabled','fake_migrate_all', 7958 'decode_credentials','driver_args', 7959 'adapter_args', 'attempts', 7960 'bigint_id','debug','lazy_tables', 7961 'do_connect'])) 7962 for table in self: 7963 db_as_dict["tables"].append(table.as_dict(flat=flat, 7964 sanitize=sanitize)) 7965 return db_as_dict
7966
7967 - def as_xml(self, sanitize=True):
7968 if not have_serializers: 7969 raise ImportError("No xml serializers available") 7970 d = self.as_dict(flat=True, sanitize=sanitize) 7971 return serializers.xml(d)
7972
7973 - def as_json(self, sanitize=True):
7974 if not have_serializers: 7975 raise ImportError("No json serializers available") 7976 d = self.as_dict(flat=True, sanitize=sanitize) 7977 return serializers.json(d)
7978
7979 - def as_yaml(self, sanitize=True):
7980 if not have_serializers: 7981 raise ImportError("No YAML serializers available") 7982 d = self.as_dict(flat=True, sanitize=sanitize) 7983 return serializers.yaml(d)
7984
7985 - def __contains__(self, tablename):
7986 try: 7987 return tablename in self.tables 7988 except AttributeError: 7989 # The instance has no .tables attribute yet 7990 return False
7991 7992 has_key = __contains__ 7993
7994 - def get(self,key,default=None):
7995 return self.__dict__.get(key,default)
7996
7997 - def __iter__(self):
7998 for tablename in self.tables: 7999 yield self[tablename]
8000
8001 - def __getitem__(self, key):
8002 return self.__getattr__(str(key))
8003
8004 - def __getattr__(self, key):
8005 if ogetattr(self,'_lazy_tables') and \ 8006 key in ogetattr(self,'_LAZY_TABLES'): 8007 tablename, fields, args = self._LAZY_TABLES.pop(key) 8008 return self.lazy_define_table(tablename,*fields,**args) 8009 return ogetattr(self, key)
8010
8011 - def __setitem__(self, key, value):
8012 osetattr(self, str(key), value)
8013
8014 - def __setattr__(self, key, value):
8015 if key[:1]!='_' and key in self: 8016 raise SyntaxError( 8017 'Object %s exists and cannot be redefined' % key) 8018 osetattr(self,key,value)
8019 8020 __delitem__ = object.__delattr__ 8021
8022 - def __repr__(self):
8023 if hasattr(self,'_uri'): 8024 return '<DAL uri="%s">' % hide_password(str(self._uri)) 8025 else: 8026 return '<DAL db_uid="%s">' % self._db_uid
8027
8028 - def smart_query(self,fields,text):
8029 return Set(self, smart_query(fields,text))
8030
8031 - def __call__(self, query=None, ignore_common_filters=None):
8032 if isinstance(query,Table): 8033 query = self._adapter.id_query(query) 8034 elif isinstance(query,Field): 8035 query = query!=None 8036 elif isinstance(query, dict): 8037 icf = query.get("ignore_common_filters") 8038 if icf: ignore_common_filters = icf 8039 return Set(self, query, ignore_common_filters=ignore_common_filters)
8040
8041 - def commit(self):
8042 self._adapter.commit()
8043
8044 - def rollback(self):
8045 self._adapter.rollback()
8046
8047 - def close(self):
8048 self._adapter.close() 8049 if self._db_uid in THREAD_LOCAL.db_instances: 8050 db_group = THREAD_LOCAL.db_instances[self._db_uid] 8051 db_group.remove(self) 8052 if not db_group: 8053 del THREAD_LOCAL.db_instances[self._db_uid]
8054
8055 - def executesql(self, query, placeholders=None, as_dict=False, 8056 fields=None, colnames=None):
8057 """ 8058 placeholders is optional and will always be None. 8059 If using raw SQL with placeholders, placeholders may be 8060 a sequence of values to be substituted in 8061 or, (if supported by the DB driver), a dictionary with keys 8062 matching named placeholders in your SQL. 8063 8064 Added 2009-12-05 "as_dict" optional argument. Will always be 8065 None when using DAL. If using raw SQL can be set to True 8066 and the results cursor returned by the DB driver will be 8067 converted to a sequence of dictionaries keyed with the db 8068 field names. Tested with SQLite but should work with any database 8069 since the cursor.description used to get field names is part of the 8070 Python dbi 2.0 specs. Results returned with as_dict=True are 8071 the same as those returned when applying .to_list() to a DAL query. 8072 8073 [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}] 8074 8075 Added 2012-08-24 "fields" and "colnames" optional arguments. If either 8076 is provided, the results cursor returned by the DB driver will be 8077 converted to a DAL Rows object using the db._adapter.parse() method. 8078 8079 The "fields" argument is a list of DAL Field objects that match the 8080 fields returned from the DB. The Field objects should be part of one or 8081 more Table objects defined on the DAL object. The "fields" list can 8082 include one or more DAL Table objects in addition to or instead of 8083 including Field objects, or it can be just a single table (not in a 8084 list). In that case, the Field objects will be extracted from the 8085 table(s). 8086 8087 Instead of specifying the "fields" argument, the "colnames" argument 8088 can be specified as a list of field names in tablename.fieldname format. 8089 Again, these should represent tables and fields defined on the DAL 8090 object. 8091 8092 It is also possible to specify both "fields" and the associated 8093 "colnames". In that case, "fields" can also include DAL Expression 8094 objects in addition to Field objects. For Field objects in "fields", 8095 the associated "colnames" must still be in tablename.fieldname format. 8096 For Expression objects in "fields", the associated "colnames" can 8097 be any arbitrary labels. 8098 8099 Note, the DAL Table objects referred to by "fields" or "colnames" can 8100 be dummy tables and do not have to represent any real tables in the 8101 database. Also, note that the "fields" and "colnames" must be in the 8102 same order as the fields in the results cursor returned from the DB. 8103 """ 8104 adapter = self._adapter 8105 if placeholders: 8106 adapter.execute(query, placeholders) 8107 else: 8108 adapter.execute(query) 8109 if as_dict: 8110 if not hasattr(adapter.cursor,'description'): 8111 raise RuntimeError("database does not support executesql(...,as_dict=True)") 8112 # Non-DAL legacy db query, converts cursor results to dict. 8113 # sequence of 7-item sequences. each sequence tells about a column. 8114 # first item is always the field name according to Python Database API specs 8115 columns = adapter.cursor.description 8116 # reduce the column info down to just the field names 8117 fields = [f[0] for f in columns] 8118 # will hold our finished resultset in a list 8119 data = adapter._fetchall() 8120 # convert the list for each row into a dictionary so it's 8121 # easier to work with. row['field_name'] rather than row[0] 8122 return [dict(zip(fields,row)) for row in data] 8123 try: 8124 data = adapter._fetchall() 8125 except: 8126 return None 8127 if fields or colnames: 8128 fields = [] if fields is None else fields 8129 if not isinstance(fields, list): 8130 fields = [fields] 8131 extracted_fields = [] 8132 for field in fields: 8133 if isinstance(field, Table): 8134 extracted_fields.extend([f for f in field]) 8135 else: 8136 extracted_fields.append(field) 8137 if not colnames: 8138 colnames = ['%s.%s' % (f.tablename, f.name) 8139 for f in extracted_fields] 8140 data = adapter.parse( 8141 data, fields=extracted_fields, colnames=colnames) 8142 return data
8143
8144 - def _remove_references_to(self, thistable):
8145 for table in self: 8146 table._referenced_by = [field for field in table._referenced_by 8147 if not field.table==thistable]
8148
8149 - def export_to_csv_file(self, ofile, *args, **kwargs):
8150 step = long(kwargs.get('max_fetch_rows,',500)) 8151 write_colnames = kwargs['write_colnames'] = \ 8152 kwargs.get("write_colnames", True) 8153 for table in self.tables: 8154 ofile.write('TABLE %s\r\n' % table) 8155 query = self._adapter.id_query(self[table]) 8156 nrows = self(query).count() 8157 kwargs['write_colnames'] = write_colnames 8158 for k in range(0,nrows,step): 8159 self(query).select(limitby=(k,k+step)).export_to_csv_file( 8160 ofile, *args, **kwargs) 8161 kwargs['write_colnames'] = False 8162 ofile.write('\r\n\r\n') 8163 ofile.write('END')
8164
8165 - def import_from_csv_file(self, ifile, id_map=None, null='<NULL>', 8166 unique='uuid', map_tablenames=None, 8167 ignore_missing_tables=False, 8168 *args, **kwargs):
8169 #if id_map is None: id_map={} 8170 id_offset = {} # only used if id_map is None 8171 map_tablenames = map_tablenames or {} 8172 for line in ifile: 8173 line = line.strip() 8174 if not line: 8175 continue 8176 elif line == 'END': 8177 return 8178 elif not line.startswith('TABLE ') or \ 8179 not line[6:] in self.tables: 8180 raise SyntaxError('invalid file format') 8181 else: 8182 tablename = line[6:] 8183 tablename = map_tablenames.get(tablename,tablename) 8184 if tablename is not None and tablename in self.tables: 8185 self[tablename].import_from_csv_file( 8186 ifile, id_map, null, unique, id_offset, 8187 *args, **kwargs) 8188 elif tablename is None or ignore_missing_tables: 8189 # skip all non-empty lines 8190 for line in ifile: 8191 if not line.strip(): 8192 break 8193 else: 8194 raise RuntimeError("Unable to import table that does not exist.\nTry db.import_from_csv_file(..., map_tablenames={'table':'othertable'},ignore_missing_tables=True)")
8195
8196 8197 -def DAL_unpickler(db_uid):
8198 return DAL('<zombie>',db_uid=db_uid)
8199
8200 -def DAL_pickler(db):
8201 return DAL_unpickler, (db._db_uid,)
8202 8203 copyreg.pickle(DAL, DAL_pickler, DAL_unpickler)
8204 8205 -class SQLALL(object):
8206 """ 8207 Helper class providing a comma-separated string having all the field names 8208 (prefixed by table name and '.') 8209 8210 normally only called from within gluon.sql 8211 """ 8212
8213 - def __init__(self, table):
8214 self._table = table
8215
8216 - def __str__(self):
8217 return ', '.join([str(field) for field in self._table])
8218
8219 # class Reference(int): 8220 -class Reference(long):
8221
8222 - def __allocate(self):
8223 if not self._record: 8224 self._record = self._table[long(self)] 8225 if not self._record: 8226 raise RuntimeError( 8227 "Using a recursive select but encountered a broken reference: %s %d"%(self._table, long(self)))
8228
8229 - def __getattr__(self, key):
8230 if key == 'id': 8231 return long(self) 8232 if key in self._table: 8233 self.__allocate() 8234 if self._record: 8235 return self._record.get(key,None) # to deal with case self.update_record() 8236 else: 8237 return None
8238
8239 - def get(self, key, default=None):
8240 return self.__getattr__(key, default)
8241
8242 - def __setattr__(self, key, value):
8243 if key.startswith('_'): 8244 long.__setattr__(self, key, value) 8245 return 8246 self.__allocate() 8247 self._record[key] = value
8248
8249 - def __getitem__(self, key):
8250 if key == 'id': 8251 return long(self) 8252 self.__allocate() 8253 return self._record.get(key, None)
8254
8255 - def __setitem__(self,key,value):
8256 self.__allocate() 8257 self._record[key] = value
8258
8259 8260 -def Reference_unpickler(data):
8261 return marshal.loads(data)
8262
8263 -def Reference_pickler(data):
8264 try: 8265 marshal_dump = marshal.dumps(long(data)) 8266 except AttributeError: 8267 marshal_dump = 'i%s' % struct.pack('<i', long(data)) 8268 return (Reference_unpickler, (marshal_dump,))
8269 8270 copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
8271 8272 -class MethodAdder(object):
8273 - def __init__(self,table):
8274 self.table = table
8275 - def __call__(self):
8276 return self.register()
8277 - def __getattr__(self,method_name):
8278 return self.register(method_name)
8279 - def register(self,method_name=None):
8280 def _decorated(f): 8281 instance = self.table 8282 import types 8283 method = types.MethodType(f, instance, instance.__class__) 8284 name = method_name or f.func_name 8285 setattr(instance, name, method) 8286 return f
8287 return _decorated
8288
8289 -class Table(object):
8290 8291 """ 8292 an instance of this class represents a database table 8293 8294 Example:: 8295 8296 db = DAL(...) 8297 db.define_table('users', Field('name')) 8298 db.users.insert(name='me') # print db.users._insert(...) to see SQL 8299 db.users.drop() 8300 """ 8301
8302 - def __init__( 8303 self, 8304 db, 8305 tablename, 8306 *fields, 8307 **args 8308 ):
8309 """ 8310 Initializes the table and performs checking on the provided fields. 8311 8312 Each table will have automatically an 'id'. 8313 8314 If a field is of type Table, the fields (excluding 'id') from that table 8315 will be used instead. 8316 8317 :raises SyntaxError: when a supplied field is of incorrect type. 8318 """ 8319 self._actual = False # set to True by define_table() 8320 self._tablename = tablename 8321 self._ot = args.get('actual_name') 8322 self._sequence_name = args.get('sequence_name') or \ 8323 db and db._adapter.sequence_name(tablename) 8324 self._trigger_name = args.get('trigger_name') or \ 8325 db and db._adapter.trigger_name(tablename) 8326 self._common_filter = args.get('common_filter') 8327 self._format = args.get('format') 8328 self._singular = args.get( 8329 'singular',tablename.replace('_',' ').capitalize()) 8330 self._plural = args.get( 8331 'plural',pluralize(self._singular.lower()).capitalize()) 8332 # horrible but for backard compatibility of appamdin: 8333 if 'primarykey' in args and args['primarykey'] is not None: 8334 self._primarykey = args.get('primarykey') 8335 8336 self._before_insert = [] 8337 self._before_update = [Set.delete_uploaded_files] 8338 self._before_delete = [Set.delete_uploaded_files] 8339 self._after_insert = [] 8340 self._after_update = [] 8341 self._after_delete = [] 8342 8343 self.add_method = MethodAdder(self) 8344 8345 fieldnames,newfields=set(),[] 8346 _primarykey = getattr(self, '_primarykey', None) 8347 if _primarykey is not None: 8348 if not isinstance(_primarykey, list): 8349 raise SyntaxError( 8350 "primarykey must be a list of fields from table '%s'" \ 8351 % tablename) 8352 if len(_primarykey)==1: 8353 self._id = [f for f in fields if isinstance(f,Field) \ 8354 and f.name==_primarykey[0]][0] 8355 elif not [f for f in fields if (isinstance(f,Field) and 8356 f.type=='id') or (isinstance(f, dict) and 8357 f.get("type", None)=="id")]: 8358 field = Field('id', 'id') 8359 newfields.append(field) 8360 fieldnames.add('id') 8361 self._id = field 8362 virtual_fields = [] 8363 def include_new(field): 8364 newfields.append(field) 8365 fieldnames.add(field.name) 8366 if field.type=='id': 8367 self._id = field
8368 for field in fields: 8369 if isinstance(field, (FieldMethod, FieldVirtual)): 8370 virtual_fields.append(field) 8371 elif isinstance(field, Field) and not field.name in fieldnames: 8372 if field.db is not None: 8373 field = copy.copy(field) 8374 include_new(field) 8375 elif isinstance(field, dict) and not field['fieldname'] in fieldnames: 8376 include_new(Field(**field)) 8377 elif isinstance(field, Table): 8378 table = field 8379 for field in table: 8380 if not field.name in fieldnames and not field.type=='id': 8381 t2 = not table._actual and self._tablename 8382 include_new(field.clone(point_self_references_to=t2)) 8383 elif not isinstance(field, (Field, Table)): 8384 raise SyntaxError( 8385 'define_table argument is not a Field or Table: %s' % field) 8386 fields = newfields 8387 self._db = db 8388 tablename = tablename 8389 self._fields = SQLCallableList() 8390 self.virtualfields = [] 8391 fields = list(fields) 8392 8393 if db and db._adapter.uploads_in_blob==True: 8394 uploadfields = [f.name for f in fields if f.type=='blob'] 8395 for field in fields: 8396 fn = field.uploadfield 8397 if isinstance(field, Field) and field.type == 'upload'\ 8398 and fn is True: 8399 fn = field.uploadfield = '%s_blob' % field.name 8400 if isinstance(fn,str) and not fn in uploadfields: 8401 fields.append(Field(fn,'blob',default='', 8402 writable=False,readable=False)) 8403 8404 lower_fieldnames = set() 8405 reserved = dir(Table) + ['fields'] 8406 if (db and db.check_reserved): 8407 check_reserved = db.check_reserved_keyword 8408 else: 8409 def check_reserved(field_name): 8410 if field_name in reserved: 8411 raise SyntaxError("field name %s not allowed" % field_name)
8412 for field in fields: 8413 field_name = field.name 8414 check_reserved(field_name) 8415 fn_lower = field_name.lower() 8416 if fn_lower in lower_fieldnames: 8417 raise SyntaxError("duplicate field %s in table %s" \ 8418 % (field_name, tablename)) 8419 else: 8420 lower_fieldnames.add(fn_lower) 8421 8422 self.fields.append(field_name) 8423 self[field_name] = field 8424 if field.type == 'id': 8425 self['id'] = field 8426 field.tablename = field._tablename = tablename 8427 field.table = field._table = self 8428 field.db = field._db = db 8429 self.ALL = SQLALL(self) 8430 8431 if _primarykey is not None: 8432 for k in _primarykey: 8433 if k not in self.fields: 8434 raise SyntaxError( 8435 "primarykey must be a list of fields from table '%s " % tablename) 8436 else: 8437 self[k].notnull = True 8438 for field in virtual_fields: 8439 self[field.name] = field 8440 8441 @property
8442 - def fields(self):
8443 return self._fields
8444
8445 - def update(self,*args,**kwargs):
8446 raise RuntimeError("Syntax Not Supported")
8447
8448 - def _enable_record_versioning(self, 8449 archive_db=None, 8450 archive_name = '%(tablename)s_archive', 8451 is_active = 'is_active', 8452 current_record = 'current_record', 8453 current_record_label = None):
8454 db = self._db 8455 archive_db = archive_db or db 8456 archive_name = archive_name % dict(tablename=self._tablename) 8457 if archive_name in archive_db.tables(): 8458 return # do not try define the archive if already exists 8459 fieldnames = self.fields() 8460 same_db = archive_db is db 8461 field_type = self if same_db else 'bigint' 8462 clones = [] 8463 for field in self: 8464 nfk = same_db or not field.type.startswith('reference') 8465 clones.append(field.clone( 8466 unique=False, type=field.type if nfk else 'bigint')) 8467 archive_db.define_table( 8468 archive_name, 8469 Field(current_record,field_type,label=current_record_label), 8470 *clones,**dict(format=self._format)) 8471 8472 self._before_update.append( 8473 lambda qset,fs,db=archive_db,an=archive_name,cn=current_record: 8474 archive_record(qset,fs,db[an],cn)) 8475 if is_active and is_active in fieldnames: 8476 self._before_delete.append( 8477 lambda qset: qset.update(is_active=False)) 8478 newquery = lambda query, t=self, name=self._tablename: \ 8479 reduce(AND,[db[tn].is_active == True 8480 for tn in db._adapter.tables(query) 8481 if tn==name or getattr(db[tn],'_ot',None)==name]) 8482 query = self._common_filter 8483 if query: 8484 newquery = query & newquery 8485 self._common_filter = newquery
8486
8487 - def _validate(self,**vars):
8488 errors = Row() 8489 for key,value in vars.iteritems(): 8490 value,error = self[key].validate(value) 8491 if error: 8492 errors[key] = error 8493 return errors
8494
8495 - def _create_references(self):
8496 db = self._db 8497 pr = db._pending_references 8498 self._referenced_by = [] 8499 self._references = [] 8500 for field in self: 8501 fieldname = field.name 8502 field_type = field.type 8503 if isinstance(field_type,str) and field_type[:10] == 'reference ': 8504 ref = field_type[10:].strip() 8505 if not ref: 8506 SyntaxError('Table: reference to nothing: %s' %ref) 8507 if '.' in ref: 8508 rtablename, throw_it,rfieldname = ref.partition('.') 8509 else: 8510 rtablename, rfieldname = ref, None 8511 if not rtablename in db: 8512 pr[rtablename] = pr.get(rtablename,[]) + [field] 8513 continue 8514 rtable = db[rtablename] 8515 if rfieldname: 8516 if not hasattr(rtable,'_primarykey'): 8517 raise SyntaxError( 8518 'keyed tables can only reference other keyed tables (for now)') 8519 if rfieldname not in rtable.fields: 8520 raise SyntaxError( 8521 "invalid field '%s' for referenced table '%s' in table '%s'" \ 8522 % (rfieldname, rtablename, self._tablename)) 8523 rfield = rtable[rfieldname] 8524 else: 8525 rfield = rtable._id 8526 rtable._referenced_by.append(field) 8527 field.referent = rfield 8528 self._references.append(field) 8529 else: 8530 field.referent = None 8531 for referee in pr.get(self._tablename,[]): 8532 self._referenced_by.append(referee)
8533
8534 - def _filter_fields(self, record, id=False):
8535 return dict([(k, v) for (k, v) in record.iteritems() if k 8536 in self.fields and (self[k].type!='id' or id)])
8537
8538 - def _build_query(self,key):
8539 """ for keyed table only """ 8540 query = None 8541 for k,v in key.iteritems(): 8542 if k in self._primarykey: 8543 if query: 8544 query = query & (self[k] == v) 8545 else: 8546 query = (self[k] == v) 8547 else: 8548 raise SyntaxError( 8549 'Field %s is not part of the primary key of %s' % \ 8550 (k,self._tablename)) 8551 return query
8552
8553 - def __getitem__(self, key):
8554 if not key: 8555 return None 8556 elif isinstance(key, dict): 8557 """ for keyed table """ 8558 query = self._build_query(key) 8559 return self._db(query).select(limitby=(0,1), orderby_on_limitby=False).first() 8560 elif str(key).isdigit() or 'google' in DRIVERS and isinstance(key, Key): 8561 return self._db(self._id == key).select(limitby=(0,1), orderby_on_limitby=False).first() 8562 elif key: 8563 return ogetattr(self, str(key))
8564
8565 - def __call__(self, key=DEFAULT, **kwargs):
8566 for_update = kwargs.get('_for_update',False) 8567 if '_for_update' in kwargs: del kwargs['_for_update'] 8568 8569 orderby = kwargs.get('_orderby',None) 8570 if '_orderby' in kwargs: del kwargs['_orderby'] 8571 8572 if not key is DEFAULT: 8573 if isinstance(key, Query): 8574 record = self._db(key).select( 8575 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first() 8576 elif not str(key).isdigit(): 8577 record = None 8578 else: 8579 record = self._db(self._id == key).select( 8580 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first() 8581 if record: 8582 for k,v in kwargs.iteritems(): 8583 if record[k]!=v: return None 8584 return record 8585 elif kwargs: 8586 query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()]) 8587 return self._db(query).select(limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first() 8588 else: 8589 return None
8590
8591 - def __setitem__(self, key, value):
8592 if isinstance(key, dict) and isinstance(value, dict): 8593 """ option for keyed table """ 8594 if set(key.keys()) == set(self._primarykey): 8595 value = self._filter_fields(value) 8596 kv = {} 8597 kv.update(value) 8598 kv.update(key) 8599 if not self.insert(**kv): 8600 query = self._build_query(key) 8601 self._db(query).update(**self._filter_fields(value)) 8602 else: 8603 raise SyntaxError( 8604 'key must have all fields from primary key: %s'%\ 8605 (self._primarykey)) 8606 elif str(key).isdigit(): 8607 if key == 0: 8608 self.insert(**self._filter_fields(value)) 8609 elif self._db(self._id == key)\ 8610 .update(**self._filter_fields(value)) is None: 8611 raise SyntaxError('No such record: %s' % key) 8612 else: 8613 if isinstance(key, dict): 8614 raise SyntaxError( 8615 'value must be a dictionary: %s' % value) 8616 osetattr(self, str(key), value)
8617 8618 __getattr__ = __getitem__ 8619
8620 - def __setattr__(self, key, value):
8621 if key[:1]!='_' and key in self: 8622 raise SyntaxError('Object exists and cannot be redefined: %s' % key) 8623 osetattr(self,key,value)
8624
8625 - def __delitem__(self, key):
8626 if isinstance(key, dict): 8627 query = self._build_query(key) 8628 if not self._db(query).delete(): 8629 raise SyntaxError('No such record: %s' % key) 8630 elif not str(key).isdigit() or \ 8631 not self._db(self._id == key).delete(): 8632 raise SyntaxError('No such record: %s' % key)
8633
8634 - def __contains__(self,key):
8635 return hasattr(self,key)
8636 8637 has_key = __contains__ 8638
8639 - def items(self):
8640 return self.__dict__.items()
8641
8642 - def __iter__(self):
8643 for fieldname in self.fields: 8644 yield self[fieldname]
8645
8646 - def iteritems(self):
8647 return self.__dict__.iteritems()
8648 8649
8650 - def __repr__(self):
8651 return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
8652
8653 - def __str__(self):
8654 if self._ot is not None: 8655 ot = self._db._adapter.QUOTE_TEMPLATE % self._ot 8656 if 'Oracle' in str(type(self._db._adapter)): 8657 return '%s %s' % (ot, self._tablename) 8658 return '%s AS %s' % (ot, self._tablename) 8659 return self._tablename
8660
8661 - def _drop(self, mode = ''):
8662 return self._db._adapter._drop(self, mode)
8663
8664 - def drop(self, mode = ''):
8665 return self._db._adapter.drop(self,mode)
8666
8667 - def _listify(self,fields,update=False):
8668 new_fields = {} # format: new_fields[name] = (field,value) 8669 8670 # store all fields passed as input in new_fields 8671 for name in fields: 8672 if not name in self.fields: 8673 if name != 'id': 8674 raise SyntaxError( 8675 'Field %s does not belong to the table' % name) 8676 else: 8677 field = self[name] 8678 value = fields[name] 8679 if field.filter_in: 8680 value = field.filter_in(value) 8681 new_fields[name] = (field,value) 8682 8683 # check all fields that should be in the table but are not passed 8684 to_compute = [] 8685 for ofield in self: 8686 name = ofield.name 8687 if not name in new_fields: 8688 # if field is supposed to be computed, compute it! 8689 if ofield.compute: # save those to compute for later 8690 to_compute.append((name,ofield)) 8691 # if field is required, check its default value 8692 elif not update and not ofield.default is None: 8693 value = ofield.default 8694 fields[name] = value 8695 new_fields[name] = (ofield,value) 8696 # if this is an update, user the update field instead 8697 elif update and not ofield.update is None: 8698 value = ofield.update 8699 fields[name] = value 8700 new_fields[name] = (ofield,value) 8701 # if the field is still not there but it should, error 8702 elif not update and ofield.required: 8703 raise RuntimeError( 8704 'Table: missing required field: %s' % name) 8705 # now deal with fields that are supposed to be computed 8706 if to_compute: 8707 row = Row(fields) 8708 for name,ofield in to_compute: 8709 # try compute it 8710 try: 8711 row[name] = new_value = ofield.compute(row) 8712 new_fields[name] = (ofield, new_value) 8713 except (KeyError, AttributeError): 8714 # error silently unless field is required! 8715 if ofield.required: 8716 raise SyntaxError('unable to compute field: %s' % name) 8717 return new_fields.values()
8718
8719 - def _attempt_upload(self, fields):
8720 for field in self: 8721 if field.type=='upload' and field.name in fields: 8722 value = fields[field.name] 8723 if value is not None and not isinstance(value,str): 8724 if hasattr(value,'file') and hasattr(value,'filename'): 8725 new_name = field.store(value.file,filename=value.filename) 8726 elif hasattr(value,'read') and hasattr(value,'name'): 8727 new_name = field.store(value,filename=value.name) 8728 else: 8729 raise RuntimeError("Unable to handle upload") 8730 fields[field.name] = new_name
8731
8732 - def _defaults(self, fields):
8733 "If there are no fields/values specified, return table defaults" 8734 if not fields: 8735 fields = {} 8736 for field in self: 8737 if field.type != "id": 8738 fields[field.name] = field.default 8739 return fields
8740
8741 - def _insert(self, **fields):
8742 fields = self._defaults(fields) 8743 return self._db._adapter._insert(self, self._listify(fields))
8744
8745 - def insert(self, **fields):
8746 fields = self._defaults(fields) 8747 self._attempt_upload(fields) 8748 if any(f(fields) for f in self._before_insert): return 0 8749 ret = self._db._adapter.insert(self, self._listify(fields)) 8750 if ret and self._after_insert: 8751 fields = Row(fields) 8752 [f(fields,ret) for f in self._after_insert] 8753 return ret
8754
8755 - def validate_and_insert(self,**fields):
8756 response = Row() 8757 response.errors = Row() 8758 new_fields = copy.copy(fields) 8759 for key,value in fields.iteritems(): 8760 value,error = self[key].validate(value) 8761 if error: 8762 response.errors[key] = "%s" % error 8763 else: 8764 new_fields[key] = value 8765 if not response.errors: 8766 response.id = self.insert(**new_fields) 8767 else: 8768 response.id = None 8769 return response
8770
8771 - def validate_and_update(self, _key=DEFAULT, **fields):
8772 response = Row() 8773 response.errors = Row() 8774 new_fields = copy.copy(fields) 8775 8776 for key,value in fields.iteritems(): 8777 value,error = self[key].validate(value) 8778 if error: 8779 response.errors[key] = "%s" % error 8780 else: 8781 new_fields[key] = value 8782 8783 if _key is DEFAULT: 8784 record = self(**values) 8785 elif isinstance(_key,dict): 8786 record = self(**_key) 8787 else: 8788 record = self(_key) 8789 8790 if not response.errors and record: 8791 row = self._db(self._id==_key) 8792 response.id = row.update(**fields) 8793 else: 8794 response.id = None 8795 return response
8796
8797 - def update_or_insert(self, _key=DEFAULT, **values):
8798 if _key is DEFAULT: 8799 record = self(**values) 8800 elif isinstance(_key,dict): 8801 record = self(**_key) 8802 else: 8803 record = self(_key) 8804 if record: 8805 record.update_record(**values) 8806 newid = None 8807 else: 8808 newid = self.insert(**values) 8809 return newid
8810
8811 - def bulk_insert(self, items):
8812 """ 8813 here items is a list of dictionaries 8814 """ 8815 items = [self._listify(item) for item in items] 8816 if any(f(item) for item in items for f in self._before_insert):return 0 8817 ret = self._db._adapter.bulk_insert(self,items) 8818 ret and [[f(item,ret[k]) for k,item in enumerate(items)] for f in self._after_insert] 8819 return ret
8820
8821 - def _truncate(self, mode = None):
8822 return self._db._adapter._truncate(self, mode)
8823
8824 - def truncate(self, mode = None):
8825 return self._db._adapter.truncate(self, mode)
8826
8827 - def import_from_csv_file( 8828 self, 8829 csvfile, 8830 id_map=None, 8831 null='<NULL>', 8832 unique='uuid', 8833 id_offset=None, # id_offset used only when id_map is None 8834 *args, **kwargs 8835 ):
8836 """ 8837 Import records from csv file. 8838 Column headers must have same names as table fields. 8839 Field 'id' is ignored. 8840 If column names read 'table.file' the 'table.' prefix is ignored. 8841 'unique' argument is a field which must be unique 8842 (typically a uuid field) 8843 'restore' argument is default False; 8844 if set True will remove old values in table first. 8845 'id_map' ff set to None will not map ids. 8846 The import will keep the id numbers in the restored table. 8847 This assumes that there is an field of type id that 8848 is integer and in incrementing order. 8849 Will keep the id numbers in restored table. 8850 """ 8851 8852 delimiter = kwargs.get('delimiter', ',') 8853 quotechar = kwargs.get('quotechar', '"') 8854 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL) 8855 restore = kwargs.get('restore', False) 8856 if restore: 8857 self._db[self].truncate() 8858 8859 reader = csv.reader(csvfile, delimiter=delimiter, 8860 quotechar=quotechar, quoting=quoting) 8861 colnames = None 8862 if isinstance(id_map, dict): 8863 if not self._tablename in id_map: 8864 id_map[self._tablename] = {} 8865 id_map_self = id_map[self._tablename] 8866 8867 def fix(field, value, id_map, id_offset): 8868 list_reference_s='list:reference' 8869 if value == null: 8870 value = None 8871 elif field.type=='blob': 8872 value = base64.b64decode(value) 8873 elif field.type=='double' or field.type=='float': 8874 if not value.strip(): 8875 value = None 8876 else: 8877 value = float(value) 8878 elif field.type in ('integer','bigint'): 8879 if not value.strip(): 8880 value = None 8881 else: 8882 value = long(value) 8883 elif field.type.startswith('list:string'): 8884 value = bar_decode_string(value) 8885 elif field.type.startswith(list_reference_s): 8886 ref_table = field.type[len(list_reference_s):].strip() 8887 if id_map is not None: 8888 value = [id_map[ref_table][long(v)] \ 8889 for v in bar_decode_string(value)] 8890 else: 8891 value = [v for v in bar_decode_string(value)] 8892 elif field.type.startswith('list:'): 8893 value = bar_decode_integer(value) 8894 elif id_map and field.type.startswith('reference'): 8895 try: 8896 value = id_map[field.type[9:].strip()][long(value)] 8897 except KeyError: 8898 pass 8899 elif id_offset and field.type.startswith('reference'): 8900 try: 8901 value = id_offset[field.type[9:].strip()]+long(value) 8902 except KeyError: 8903 pass 8904 return (field.name, value)
8905 8906 def is_id(colname): 8907 if colname in self: 8908 return self[colname].type == 'id' 8909 else: 8910 return False 8911 8912 first = True 8913 unique_idx = None 8914 for lineno, line in enumerate(reader): 8915 if not line: 8916 break 8917 if not colnames: 8918 # assume this is the first line of the input, contains colnames 8919 colnames = [x.split('.',1)[-1] for x in line][:len(line)] 8920 cols, cid = [], None 8921 for i,colname in enumerate(colnames): 8922 if is_id(colname): 8923 cid = i 8924 elif colname in self.fields: 8925 cols.append((i,self[colname])) 8926 if colname == unique: 8927 unique_idx = i 8928 else: 8929 # every other line contains instead data 8930 items = [] 8931 for i, field in cols: 8932 try: 8933 items.append(fix(field, line[i], id_map, id_offset)) 8934 except ValueError: 8935 raise RuntimeError("Unable to parse line:%s field:%s value:'%s'" 8936 % (lineno+1,field,line[i])) 8937 8938 if not (id_map or cid is None or id_offset is None or unique_idx): 8939 csv_id = long(line[cid]) 8940 curr_id = self.insert(**dict(items)) 8941 if first: 8942 first = False 8943 # First curr_id is bigger than csv_id, 8944 # then we are not restoring but 8945 # extending db table with csv db table 8946 id_offset[self._tablename] = (curr_id-csv_id) \ 8947 if curr_id>csv_id else 0 8948 # create new id until we get the same as old_id+offset 8949 while curr_id<csv_id+id_offset[self._tablename]: 8950 self._db(self._db[self][colnames[cid]] == curr_id).delete() 8951 curr_id = self.insert(**dict(items)) 8952 # Validation. Check for duplicate of 'unique' &, 8953 # if present, update instead of insert. 8954 elif not unique_idx: 8955 new_id = self.insert(**dict(items)) 8956 else: 8957 unique_value = line[unique_idx] 8958 query = self._db[self][unique] == unique_value 8959 record = self._db(query).select().first() 8960 if record: 8961 record.update_record(**dict(items)) 8962 new_id = record[self._id.name] 8963 else: 8964 new_id = self.insert(**dict(items)) 8965 if id_map and cid is not None: 8966 id_map_self[long(line[cid])] = new_id 8967
8968 - def as_dict(self, flat=False, sanitize=True):
8969 table_as_dict = dict(tablename=str(self), fields=[], 8970 sequence_name=self._sequence_name, 8971 trigger_name=self._trigger_name, 8972 common_filter=self._common_filter, format=self._format, 8973 singular=self._singular, plural=self._plural) 8974 8975 for field in self: 8976 if (field.readable or field.writable) or (not sanitize): 8977 table_as_dict["fields"].append(field.as_dict( 8978 flat=flat, sanitize=sanitize)) 8979 return table_as_dict
8980
8981 - def as_xml(self, sanitize=True):
8982 if not have_serializers: 8983 raise ImportError("No xml serializers available") 8984 d = self.as_dict(flat=True, sanitize=sanitize) 8985 return serializers.xml(d)
8986
8987 - def as_json(self, sanitize=True):
8988 if not have_serializers: 8989 raise ImportError("No json serializers available") 8990 d = self.as_dict(flat=True, sanitize=sanitize) 8991 return serializers.json(d)
8992
8993 - def as_yaml(self, sanitize=True):
8994 if not have_serializers: 8995 raise ImportError("No YAML serializers available") 8996 d = self.as_dict(flat=True, sanitize=sanitize) 8997 return serializers.yaml(d)
8998
8999 - def with_alias(self, alias):
9000 return self._db._adapter.alias(self,alias)
9001
9002 - def on(self, query):
9003 return Expression(self._db,self._db._adapter.ON,self,query)
9004
9005 -def archive_record(qset,fs,archive_table,current_record):
9006 tablenames = qset.db._adapter.tables(qset.query) 9007 if len(tablenames)!=1: raise RuntimeError("cannot update join") 9008 table = qset.db[tablenames[0]] 9009 for row in qset.select(): 9010 fields = archive_table._filter_fields(row) 9011 fields[current_record] = row.id 9012 archive_table.insert(**fields) 9013 return False
9014
9015 9016 9017 -class Expression(object):
9018
9019 - def __init__( 9020 self, 9021 db, 9022 op, 9023 first=None, 9024 second=None, 9025 type=None, 9026 **optional_args 9027 ):
9028 9029 self.db = db 9030 self.op = op 9031 self.first = first 9032 self.second = second 9033 self._table = getattr(first,'_table',None) 9034 ### self._tablename = first._tablename ## CHECK 9035 if not type and first and hasattr(first,'type'): 9036 self.type = first.type 9037 else: 9038 self.type = type 9039 self.optional_args = optional_args
9040
9041 - def sum(self):
9042 db = self.db 9043 return Expression(db, db._adapter.AGGREGATE, self, 'SUM', self.type)
9044
9045 - def max(self):
9046 db = self.db 9047 return Expression(db, db._adapter.AGGREGATE, self, 'MAX', self.type)
9048
9049 - def min(self):
9050 db = self.db 9051 return Expression(db, db._adapter.AGGREGATE, self, 'MIN', self.type)
9052
9053 - def len(self):
9054 db = self.db 9055 return Expression(db, db._adapter.LENGTH, self, None, 'integer')
9056
9057 - def avg(self):
9058 db = self.db 9059 return Expression(db, db._adapter.AGGREGATE, self, 'AVG', self.type)
9060
9061 - def abs(self):
9062 db = self.db 9063 return Expression(db, db._adapter.AGGREGATE, self, 'ABS', self.type)
9064
9065 - def lower(self):
9066 db = self.db 9067 return Expression(db, db._adapter.LOWER, self, None, self.type)
9068
9069 - def upper(self):
9070 db = self.db 9071 return Expression(db, db._adapter.UPPER, self, None, self.type)
9072
9073 - def replace(self,a,b):
9074 db = self.db 9075 return Expression(db, db._adapter.REPLACE, self, (a,b), self.type)
9076
9077 - def year(self):
9078 db = self.db 9079 return Expression(db, db._adapter.EXTRACT, self, 'year', 'integer')
9080
9081 - def month(self):
9082 db = self.db 9083 return Expression(db, db._adapter.EXTRACT, self, 'month', 'integer')
9084
9085 - def day(self):
9086 db = self.db 9087 return Expression(db, db._adapter.EXTRACT, self, 'day', 'integer')
9088
9089 - def hour(self):
9090 db = self.db 9091 return Expression(db, db._adapter.EXTRACT, self, 'hour', 'integer')
9092
9093 - def minutes(self):
9094 db = self.db 9095 return Expression(db, db._adapter.EXTRACT, self, 'minute', 'integer')
9096
9097 - def coalesce(self,*others):
9098 db = self.db 9099 return Expression(db, db._adapter.COALESCE, self, others, self.type)
9100
9101 - def coalesce_zero(self):
9102 db = self.db 9103 return Expression(db, db._adapter.COALESCE_ZERO, self, None, self.type)
9104
9105 - def seconds(self):
9106 db = self.db 9107 return Expression(db, db._adapter.EXTRACT, self, 'second', 'integer')
9108
9109 - def epoch(self):
9110 db = self.db 9111 return Expression(db, db._adapter.EPOCH, self, None, 'integer')
9112
9113 - def __getslice__(self, start, stop):
9114 db = self.db 9115 if start < 0: 9116 pos0 = '(%s - %d)' % (self.len(), abs(start) - 1) 9117 else: 9118 pos0 = start + 1 9119 9120 if stop < 0: 9121 length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0) 9122 elif stop == sys.maxint: 9123 length = self.len() 9124 else: 9125 length = '(%s - %s)' % (stop + 1, pos0) 9126 return Expression(db,db._adapter.SUBSTRING, 9127 self, (pos0, length), self.type)
9128
9129 - def __getitem__(self, i):
9130 return self[i:i + 1]
9131
9132 - def __str__(self):
9133 return self.db._adapter.expand(self,self.type)
9134
9135 - def __or__(self, other): # for use in sortby
9136 db = self.db 9137 return Expression(db,db._adapter.COMMA,self,other,self.type)
9138
9139 - def __invert__(self):
9140 db = self.db 9141 if hasattr(self,'_op') and self.op == db._adapter.INVERT: 9142 return self.first 9143 return Expression(db,db._adapter.INVERT,self,type=self.type)
9144
9145 - def __add__(self, other):
9146 db = self.db 9147 return Expression(db,db._adapter.ADD,self,other,self.type)
9148
9149 - def __sub__(self, other):
9150 db = self.db 9151 if self.type in ('integer','bigint'): 9152 result_type = 'integer' 9153 elif self.type in ['date','time','datetime','double','float']: 9154 result_type = 'double' 9155 elif self.type.startswith('decimal('): 9156 result_type = self.type 9157 else: 9158 raise SyntaxError("subtraction operation not supported for type") 9159 return Expression(db,db._adapter.SUB,self,other,result_type)
9160
9161 - def __mul__(self, other):
9162 db = self.db 9163 return Expression(db,db._adapter.MUL,self,other,self.type)
9164
9165 - def __div__(self, other):
9166 db = self.db 9167 return Expression(db,db._adapter.DIV,self,other,self.type)
9168
9169 - def __mod__(self, other):
9170 db = self.db 9171 return Expression(db,db._adapter.MOD,self,other,self.type)
9172
9173 - def __eq__(self, value):
9174 db = self.db 9175 return Query(db, db._adapter.EQ, self, value)
9176
9177 - def __ne__(self, value):
9178 db = self.db 9179 return Query(db, db._adapter.NE, self, value)
9180
9181 - def __lt__(self, value):
9182 db = self.db 9183 return Query(db, db._adapter.LT, self, value)
9184
9185 - def __le__(self, value):
9186 db = self.db 9187 return Query(db, db._adapter.LE, self, value)
9188
9189 - def __gt__(self, value):
9190 db = self.db 9191 return Query(db, db._adapter.GT, self, value)
9192
9193 - def __ge__(self, value):
9194 db = self.db 9195 return Query(db, db._adapter.GE, self, value)
9196
9197 - def like(self, value, case_sensitive=False):
9198 db = self.db 9199 op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE 9200 return Query(db, op, self, value)
9201
9202 - def regexp(self, value):
9203 db = self.db 9204 return Query(db, db._adapter.REGEXP, self, value)
9205
9206 - def belongs(self, *value, **kwattr):
9207 """ 9208 Accepts the following inputs: 9209 field.belongs(1,2) 9210 field.belongs((1,2)) 9211 field.belongs(query) 9212 9213 Does NOT accept: 9214 field.belongs(1) 9215 """ 9216 db = self.db 9217 if len(value) == 1: 9218 value = value[0] 9219 if isinstance(value,Query): 9220 value = db(value)._select(value.first._table._id) 9221 elif not isinstance(value, basestring): 9222 value = set(value) 9223 if kwattr.get('null') and None in value: 9224 value.remove(None) 9225 return (self == None) | Query(db, db._adapter.BELONGS, self, value) 9226 return Query(db, db._adapter.BELONGS, self, value)
9227
9228 - def startswith(self, value):
9229 db = self.db 9230 if not self.type in ('string', 'text', 'json', 'upload'): 9231 raise SyntaxError("startswith used with incompatible field type") 9232 return Query(db, db._adapter.STARTSWITH, self, value)
9233
9234 - def endswith(self, value):
9235 db = self.db 9236 if not self.type in ('string', 'text', 'json', 'upload'): 9237 raise SyntaxError("endswith used with incompatible field type") 9238 return Query(db, db._adapter.ENDSWITH, self, value)
9239
9240 - def contains(self, value, all=False, case_sensitive=False):
9241 """ 9242 The case_sensitive parameters is only useful for PostgreSQL 9243 For other RDMBs it is ignored and contains is always case in-sensitive 9244 For MongoDB and GAE contains is always case sensitive 9245 """ 9246 db = self.db 9247 if isinstance(value,(list, tuple)): 9248 subqueries = [self.contains(str(v).strip(),case_sensitive=case_sensitive) 9249 for v in value if str(v).strip()] 9250 if not subqueries: 9251 return self.contains('') 9252 else: 9253 return reduce(all and AND or OR,subqueries) 9254 if not self.type in ('string', 'text', 'json', 'upload') and not self.type.startswith('list:'): 9255 raise SyntaxError("contains used with incompatible field type") 9256 return Query(db, db._adapter.CONTAINS, self, value, case_sensitive=case_sensitive)
9257
9258 - def with_alias(self, alias):
9259 db = self.db 9260 return Expression(db, db._adapter.AS, self, alias, self.type)
9261 9262 # GIS expressions 9263
9264 - def st_asgeojson(self, precision=15, options=0, version=1):
9265 return Expression(self.db, self.db._adapter.ST_ASGEOJSON, self, 9266 dict(precision=precision, options=options, 9267 version=version), 'string')
9268
9269 - def st_astext(self):
9270 db = self.db 9271 return Expression(db, db._adapter.ST_ASTEXT, self, type='string')
9272
9273 - def st_x(self):
9274 db = self.db 9275 return Expression(db, db._adapter.ST_X, self, type='string')
9276
9277 - def st_y(self):
9278 db = self.db 9279 return Expression(db, db._adapter.ST_Y, self, type='string')
9280
9281 - def st_distance(self, other):
9282 db = self.db 9283 return Expression(db,db._adapter.ST_DISTANCE,self,other, 'double')
9284
9285 - def st_simplify(self, value):
9286 db = self.db 9287 return Expression(db, db._adapter.ST_SIMPLIFY, self, value, self.type)
9288 9289 # GIS queries 9290
9291 - def st_contains(self, value):
9292 db = self.db 9293 return Query(db, db._adapter.ST_CONTAINS, self, value)
9294
9295 - def st_equals(self, value):
9296 db = self.db 9297 return Query(db, db._adapter.ST_EQUALS, self, value)
9298
9299 - def st_intersects(self, value):
9300 db = self.db 9301 return Query(db, db._adapter.ST_INTERSECTS, self, value)
9302
9303 - def st_overlaps(self, value):
9304 db = self.db 9305 return Query(db, db._adapter.ST_OVERLAPS, self, value)
9306
9307 - def st_touches(self, value):
9308 db = self.db 9309 return Query(db, db._adapter.ST_TOUCHES, self, value)
9310
9311 - def st_within(self, value):
9312 db = self.db 9313 return Query(db, db._adapter.ST_WITHIN, self, value)
9314
9315 # for use in both Query and sortby 9316 9317 9318 -class SQLCustomType(object):
9319 """ 9320 allows defining of custom SQL types 9321 9322 Example:: 9323 9324 decimal = SQLCustomType( 9325 type ='double', 9326 native ='integer', 9327 encoder =(lambda x: int(float(x) * 100)), 9328 decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) ) 9329 ) 9330 9331 db.define_table( 9332 'example', 9333 Field('value', type=decimal) 9334 ) 9335 9336 :param type: the web2py type (default = 'string') 9337 :param native: the backend type 9338 :param encoder: how to encode the value to store it in the backend 9339 :param decoder: how to decode the value retrieved from the backend 9340 :param validator: what validators to use ( default = None, will use the 9341 default validator for type) 9342 """ 9343
9344 - def __init__( 9345 self, 9346 type='string', 9347 native=None, 9348 encoder=None, 9349 decoder=None, 9350 validator=None, 9351 _class=None, 9352 ):
9353 9354 self.type = type 9355 self.native = native 9356 self.encoder = encoder or (lambda x: x) 9357 self.decoder = decoder or (lambda x: x) 9358 self.validator = validator 9359 self._class = _class or type
9360
9361 - def startswith(self, text=None):
9362 try: 9363 return self.type.startswith(self, text) 9364 except TypeError: 9365 return False
9366
9367 - def __getslice__(self, a=0, b=100):
9368 return None
9369
9370 - def __getitem__(self, i):
9371 return None
9372
9373 - def __str__(self):
9374 return self._class
9375
9376 -class FieldVirtual(object):
9377 - def __init__(self, name, f=None, ftype='string',label=None,table_name=None):
9378 # for backward compatibility 9379 (self.name, self.f) = (name, f) if f else ('unknown', name) 9380 self.type = ftype 9381 self.label = label or self.name.capitalize().replace('_',' ') 9382 self.represent = lambda v,r:v 9383 self.formatter = IDENTITY 9384 self.comment = None 9385 self.readable = True 9386 self.writable = False 9387 self.requires = None 9388 self.widget = None 9389 self.tablename = table_name 9390 self.filter_out = None
9391 - def __str__(self):
9392 return '%s.%s' % (self.tablename, self.name)
9393
9394 -class FieldMethod(object):
9395 - def __init__(self, name, f=None, handler=None):
9396 # for backward compatibility 9397 (self.name, self.f) = (name, f) if f else ('unknown', name) 9398 self.handler = handler
9399
9400 -def list_represent(x,r=None):
9401 return ', '.join(str(y) for y in x or [])
9402
9403 -class Field(Expression):
9404 9405 Virtual = FieldVirtual 9406 Method = FieldMethod 9407 Lazy = FieldMethod # for backward compatibility 9408 9409 """ 9410 an instance of this class represents a database field 9411 9412 example:: 9413 9414 a = Field(name, 'string', length=32, default=None, required=False, 9415 requires=IS_NOT_EMPTY(), ondelete='CASCADE', 9416 notnull=False, unique=False, 9417 uploadfield=True, widget=None, label=None, comment=None, 9418 uploadfield=True, # True means store on disk, 9419 # 'a_field_name' means store in this field in db 9420 # False means file content will be discarded. 9421 writable=True, readable=True, update=None, authorize=None, 9422 autodelete=False, represent=None, uploadfolder=None, 9423 uploadseparate=False # upload to separate directories by uuid_keys 9424 # first 2 character and tablename.fieldname 9425 # False - old behavior 9426 # True - put uploaded file in 9427 # <uploaddir>/<tablename>.<fieldname>/uuid_key[:2] 9428 # directory) 9429 uploadfs=None # a pyfilesystem where to store upload 9430 9431 to be used as argument of DAL.define_table 9432 9433 allowed field types: 9434 string, boolean, integer, double, text, blob, 9435 date, time, datetime, upload, password 9436 9437 """ 9438
9439 - def __init__( 9440 self, 9441 fieldname, 9442 type='string', 9443 length=None, 9444 default=DEFAULT, 9445 required=False, 9446 requires=DEFAULT, 9447 ondelete='CASCADE', 9448 notnull=False, 9449 unique=False, 9450 uploadfield=True, 9451 widget=None, 9452 label=None, 9453 comment=None, 9454 writable=True, 9455 readable=True, 9456 update=None, 9457 authorize=None, 9458 autodelete=False, 9459 represent=None, 9460 uploadfolder=None, 9461 uploadseparate=False, 9462 uploadfs=None, 9463 compute=None, 9464 custom_store=None, 9465 custom_retrieve=None, 9466 custom_retrieve_file_properties=None, 9467 custom_delete=None, 9468 filter_in = None, 9469 filter_out = None, 9470 custom_qualifier = None, 9471 map_none = None, 9472 ):
9473 self._db = self.db = None # both for backward compatibility 9474 self.op = None 9475 self.first = None 9476 self.second = None 9477 if isinstance(fieldname, unicode): 9478 try: 9479 fieldname = str(fieldname) 9480 except UnicodeEncodeError: 9481 raise SyntaxError('Field: invalid unicode field name') 9482 self.name = fieldname = cleanup(fieldname) 9483 if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \ 9484 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname): 9485 raise SyntaxError('Field: invalid field name: %s' % fieldname) 9486 self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type 9487 self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512) 9488 self.default = default if default!=DEFAULT else (update or None) 9489 self.required = required # is this field required 9490 self.ondelete = ondelete.upper() # this is for reference fields only 9491 self.notnull = notnull 9492 self.unique = unique 9493 self.uploadfield = uploadfield 9494 self.uploadfolder = uploadfolder 9495 self.uploadseparate = uploadseparate 9496 self.uploadfs = uploadfs 9497 self.widget = widget 9498 self.comment = comment 9499 self.writable = writable 9500 self.readable = readable 9501 self.update = update 9502 self.authorize = authorize 9503 self.autodelete = autodelete 9504 self.represent = list_represent if \ 9505 represent==None and type in ('list:integer','list:string') else represent 9506 self.compute = compute 9507 self.isattachment = True 9508 self.custom_store = custom_store 9509 self.custom_retrieve = custom_retrieve 9510 self.custom_retrieve_file_properties = custom_retrieve_file_properties 9511 self.custom_delete = custom_delete 9512 self.filter_in = filter_in 9513 self.filter_out = filter_out 9514 self.custom_qualifier = custom_qualifier 9515 self.label = label if label!=None else fieldname.replace('_',' ').title() 9516 self.requires = requires if requires!=None else [] 9517 self.map_none = map_none
9518
9519 - def set_attributes(self,*args,**attributes):
9520 self.__dict__.update(*args,**attributes)
9521
9522 - def clone(self,point_self_references_to=False,**args):
9523 field = copy.copy(self) 9524 if point_self_references_to and \ 9525 field.type == 'reference %s'+field._tablename: 9526 field.type = 'reference %s' % point_self_references_to 9527 field.__dict__.update(args) 9528 return field
9529
9530 - def store(self, file, filename=None, path=None):
9531 if self.custom_store: 9532 return self.custom_store(file,filename,path) 9533 if isinstance(file, cgi.FieldStorage): 9534 filename = filename or file.filename 9535 file = file.file 9536 elif not filename: 9537 filename = file.name 9538 filename = os.path.basename(filename.replace('/', os.sep)\ 9539 .replace('\\', os.sep)) 9540 m = REGEX_STORE_PATTERN.search(filename) 9541 extension = m and m.group('e') or 'txt' 9542 uuid_key = web2py_uuid().replace('-', '')[-16:] 9543 encoded_filename = base64.b16encode(filename).lower() 9544 newfilename = '%s.%s.%s.%s' % \ 9545 (self._tablename, self.name, uuid_key, encoded_filename) 9546 newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension 9547 self_uploadfield = self.uploadfield 9548 if isinstance(self_uploadfield,Field): 9549 blob_uploadfield_name = self_uploadfield.uploadfield 9550 keys={self_uploadfield.name: newfilename, 9551 blob_uploadfield_name: file.read()} 9552 self_uploadfield.table.insert(**keys) 9553 elif self_uploadfield == True: 9554 if path: 9555 pass 9556 elif self.uploadfolder: 9557 path = self.uploadfolder 9558 elif self.db._adapter.folder: 9559 path = pjoin(self.db._adapter.folder, '..', 'uploads') 9560 else: 9561 raise RuntimeError( 9562 "you must specify a Field(...,uploadfolder=...)") 9563 if self.uploadseparate: 9564 if self.uploadfs: 9565 raise RuntimeError("not supported") 9566 path = pjoin(path,"%s.%s" %(self._tablename, self.name), 9567 uuid_key[:2]) 9568 if not exists(path): 9569 os.makedirs(path) 9570 pathfilename = pjoin(path, newfilename) 9571 if self.uploadfs: 9572 dest_file = self.uploadfs.open(newfilename, 'wb') 9573 else: 9574 dest_file = open(pathfilename, 'wb') 9575 try: 9576 shutil.copyfileobj(file, dest_file) 9577 except IOError: 9578 raise IOError( 9579 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename) 9580 dest_file.close() 9581 return newfilename
9582
9583 - def retrieve(self, name, path=None, nameonly=False):
9584 """ 9585 if nameonly==True return (filename, fullfilename) instead of 9586 (filename, stream) 9587 """ 9588 self_uploadfield = self.uploadfield 9589 if self.custom_retrieve: 9590 return self.custom_retrieve(name, path) 9591 import http 9592 if self.authorize or isinstance(self_uploadfield, str): 9593 row = self.db(self == name).select().first() 9594 if not row: 9595 raise http.HTTP(404) 9596 if self.authorize and not self.authorize(row): 9597 raise http.HTTP(403) 9598 file_properties = self.retrieve_file_properties(name,path) 9599 filename = file_properties['filename'] 9600 if isinstance(self_uploadfield, str): # ## if file is in DB 9601 stream = StringIO.StringIO(row[self_uploadfield] or '') 9602 elif isinstance(self_uploadfield,Field): 9603 blob_uploadfield_name = self_uploadfield.uploadfield 9604 query = self_uploadfield == name 9605 data = self_uploadfield.table(query)[blob_uploadfield_name] 9606 stream = StringIO.StringIO(data) 9607 elif self.uploadfs: 9608 # ## if file is on pyfilesystem 9609 stream = self.uploadfs.open(name, 'rb') 9610 else: 9611 # ## if file is on regular filesystem 9612 # this is intentially a sting with filename and not a stream 9613 # this propagates and allows stream_file_or_304_or_206 to be called 9614 fullname = pjoin(file_properties['path'],name) 9615 if nameonly: 9616 return (filename, fullname) 9617 stream = open(fullname,'rb') 9618 return (filename, stream)
9619
9620 - def retrieve_file_properties(self, name, path=None):
9621 m = REGEX_UPLOAD_PATTERN.match(name) 9622 if not m or not self.isattachment: 9623 raise TypeError('Can\'t retrieve %s file properties' % name) 9624 self_uploadfield = self.uploadfield 9625 if self.custom_retrieve_file_properties: 9626 return self.custom_retrieve_file_properties(name, path) 9627 if m.group('name'): 9628 try: 9629 filename = base64.b16decode(m.group('name'), True) 9630 filename = REGEX_CLEANUP_FN.sub('_', filename) 9631 except (TypeError, AttributeError): 9632 filename = name 9633 else: 9634 filename = name 9635 # ## if file is in DB 9636 if isinstance(self_uploadfield, (str, Field)): 9637 return dict(path=None,filename=filename) 9638 # ## if file is on filesystem 9639 if not path: 9640 if self.uploadfolder: 9641 path = self.uploadfolder 9642 else: 9643 path = pjoin(self.db._adapter.folder, '..', 'uploads') 9644 if self.uploadseparate: 9645 t = m.group('table') 9646 f = m.group('field') 9647 u = m.group('uuidkey') 9648 path = pjoin(path,"%s.%s" % (t,f),u[:2]) 9649 return dict(path=path,filename=filename)
9650 9651
9652 - def formatter(self, value):
9653 requires = self.requires 9654 if value is None or not requires: 9655 return value or self.map_none 9656 if not isinstance(requires, (list, tuple)): 9657 requires = [requires] 9658 elif isinstance(requires, tuple): 9659 requires = list(requires) 9660 else: 9661 requires = copy.copy(requires) 9662 requires.reverse() 9663 for item in requires: 9664 if hasattr(item, 'formatter'): 9665 value = item.formatter(value) 9666 return value
9667
9668 - def validate(self, value):
9669 if not self.requires or self.requires == DEFAULT: 9670 return ((value if value!=self.map_none else None), None) 9671 requires = self.requires 9672 if not isinstance(requires, (list, tuple)): 9673 requires = [requires] 9674 for validator in requires: 9675 (value, error) = validator(value) 9676 if error: 9677 return (value, error) 9678 return ((value if value!=self.map_none else None), None)
9679
9680 - def count(self, distinct=None):
9681 return Expression(self.db, self.db._adapter.COUNT, self, distinct, 'integer')
9682
9683 - def as_dict(self, flat=False, sanitize=True):
9684 attrs = ("name", 'authorize', 'represent', 'ondelete', 9685 'custom_store', 'autodelete', 'custom_retrieve', 9686 'filter_out', 'uploadseparate', 'widget', 'uploadfs', 9687 'update', 'custom_delete', 'uploadfield', 'uploadfolder', 9688 'custom_qualifier', 'unique', 'writable', 'compute', 9689 'map_none', 'default', 'type', 'required', 'readable', 9690 'requires', 'comment', 'label', 'length', 'notnull', 9691 'custom_retrieve_file_properties', 'filter_in') 9692 serializable = (int, long, basestring, float, tuple, 9693 bool, type(None)) 9694 9695 def flatten(obj): 9696 if isinstance(obj, dict): 9697 return dict((flatten(k), flatten(v)) for k, v in 9698 obj.items()) 9699 elif isinstance(obj, (tuple, list, set)): 9700 return [flatten(v) for v in obj] 9701 elif isinstance(obj, serializable): 9702 return obj 9703 elif isinstance(obj, (datetime.datetime, 9704 datetime.date, datetime.time)): 9705 return str(obj) 9706 else: 9707 return None
9708 9709 d = dict() 9710 if not (sanitize and not (self.readable or self.writable)): 9711 for attr in attrs: 9712 if flat: 9713 d.update({attr: flatten(getattr(self, attr))}) 9714 else: 9715 d.update({attr: getattr(self, attr)}) 9716 d["fieldname"] = d.pop("name") 9717 return d
9718
9719 - def as_xml(self, sanitize=True):
9720 if have_serializers: 9721 xml = serializers.xml 9722 else: 9723 raise ImportError("No xml serializers available") 9724 d = self.as_dict(flat=True, sanitize=sanitize) 9725 return xml(d)
9726
9727 - def as_json(self, sanitize=True):
9728 if have_serializers: 9729 json = serializers.json 9730 else: 9731 raise ImportError("No json serializers available") 9732 d = self.as_dict(flat=True, sanitize=sanitize) 9733 return json(d)
9734
9735 - def as_yaml(self, sanitize=True):
9736 if have_serializers: 9737 d = self.as_dict(flat=True, sanitize=sanitize) 9738 return serializers.yaml(d) 9739 else: 9740 raise ImportError("No YAML serializers available")
9741
9742 - def __nonzero__(self):
9743 return True
9744
9745 - def __str__(self):
9746 try: 9747 return '%s.%s' % (self.tablename, self.name) 9748 except: 9749 return '<no table>.%s' % self.name
9750
9751 9752 -class Query(object):
9753 9754 """ 9755 a query object necessary to define a set. 9756 it can be stored or can be passed to DAL.__call__() to obtain a Set 9757 9758 Example:: 9759 9760 query = db.users.name=='Max' 9761 set = db(query) 9762 records = set.select() 9763 9764 """ 9765
9766 - def __init__( 9767 self, 9768 db, 9769 op, 9770 first=None, 9771 second=None, 9772 ignore_common_filters = False, 9773 **optional_args 9774 ):
9775 self.db = self._db = db 9776 self.op = op 9777 self.first = first 9778 self.second = second 9779 self.ignore_common_filters = ignore_common_filters 9780 self.optional_args = optional_args
9781
9782 - def __repr__(self):
9783 return '<Query %s>' % BaseAdapter.expand(self.db._adapter,self)
9784
9785 - def __str__(self):
9786 return self.db._adapter.expand(self)
9787
9788 - def __and__(self, other):
9789 return Query(self.db,self.db._adapter.AND,self,other)
9790 9791 __rand__ = __and__ 9792
9793 - def __or__(self, other):
9794 return Query(self.db,self.db._adapter.OR,self,other)
9795 9796 __ror__ = __or__ 9797
9798 - def __invert__(self):
9799 if self.op==self.db._adapter.NOT: 9800 return self.first 9801 return Query(self.db,self.db._adapter.NOT,self)
9802
9803 - def __eq__(self, other):
9804 return repr(self) == repr(other)
9805
9806 - def __ne__(self, other):
9807 return not (self == other)
9808
9809 - def case(self,t=1,f=0):
9810 return self.db._adapter.CASE(self,t,f)
9811
9812 - def as_dict(self, flat=False, sanitize=True):
9813 """Experimental stuff 9814 9815 This allows to return a plain dictionary with the basic 9816 query representation. Can be used with json/xml services 9817 for client-side db I/O 9818 9819 Example: 9820 >>> q = db.auth_user.id != 0 9821 >>> q.as_dict(flat=True) 9822 {"op": "NE", "first":{"tablename": "auth_user", 9823 "fieldname": "id"}, 9824 "second":0} 9825 """ 9826 9827 SERIALIZABLE_TYPES = (tuple, dict, set, list, int, long, float, 9828 basestring, type(None), bool) 9829 def loop(d): 9830 newd = dict() 9831 for k, v in d.items(): 9832 if k in ("first", "second"): 9833 if isinstance(v, self.__class__): 9834 newd[k] = loop(v.__dict__) 9835 elif isinstance(v, Field): 9836 newd[k] = {"tablename": v._tablename, 9837 "fieldname": v.name} 9838 elif isinstance(v, Expression): 9839 newd[k] = loop(v.__dict__) 9840 elif isinstance(v, SERIALIZABLE_TYPES): 9841 newd[k] = v 9842 elif isinstance(v, (datetime.date, 9843 datetime.time, 9844 datetime.datetime)): 9845 newd[k] = unicode(v) 9846 elif k == "op": 9847 if callable(v): 9848 newd[k] = v.__name__ 9849 elif isinstance(v, basestring): 9850 newd[k] = v 9851 else: pass # not callable or string 9852 elif isinstance(v, SERIALIZABLE_TYPES): 9853 if isinstance(v, dict): 9854 newd[k] = loop(v) 9855 else: newd[k] = v 9856 return newd
9857 9858 if flat: 9859 return loop(self.__dict__) 9860 else: return self.__dict__
9861 9862
9863 - def as_xml(self, sanitize=True):
9864 if have_serializers: 9865 xml = serializers.xml 9866 else: 9867 raise ImportError("No xml serializers available") 9868 d = self.as_dict(flat=True, sanitize=sanitize) 9869 return xml(d)
9870
9871 - def as_json(self, sanitize=True):
9872 if have_serializers: 9873 json = serializers.json 9874 else: 9875 raise ImportError("No json serializers available") 9876 d = self.as_dict(flat=True, sanitize=sanitize) 9877 return json(d)
9878
9879 -def xorify(orderby):
9880 if not orderby: 9881 return None 9882 orderby2 = orderby[0] 9883 for item in orderby[1:]: 9884 orderby2 = orderby2 | item 9885 return orderby2
9886
9887 -def use_common_filters(query):
9888 return (query and hasattr(query,'ignore_common_filters') and \ 9889 not query.ignore_common_filters)
9890
9891 -class Set(object):
9892 9893 """ 9894 a Set represents a set of records in the database, 9895 the records are identified by the query=Query(...) object. 9896 normally the Set is generated by DAL.__call__(Query(...)) 9897 9898 given a set, for example 9899 set = db(db.users.name=='Max') 9900 you can: 9901 set.update(db.users.name='Massimo') 9902 set.delete() # all elements in the set 9903 set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10)) 9904 and take subsets: 9905 subset = set(db.users.id<5) 9906 """ 9907
9908 - def __init__(self, db, query, ignore_common_filters = None):
9909 self.db = db 9910 self._db = db # for backward compatibility 9911 self.dquery = None 9912 9913 # if query is a dict, parse it 9914 if isinstance(query, dict): 9915 query = self.parse(query) 9916 9917 if not ignore_common_filters is None and \ 9918 use_common_filters(query) == ignore_common_filters: 9919 query = copy.copy(query) 9920 query.ignore_common_filters = ignore_common_filters 9921 self.query = query
9922
9923 - def __repr__(self):
9924 return '<Set %s>' % BaseAdapter.expand(self.db._adapter,self.query)
9925
9926 - def __call__(self, query, ignore_common_filters=False):
9927 if query is None: 9928 return self 9929 elif isinstance(query,Table): 9930 query = self.db._adapter.id_query(query) 9931 elif isinstance(query,str): 9932 query = Expression(self.db,query) 9933 elif isinstance(query,Field): 9934 query = query!=None 9935 if self.query: 9936 return Set(self.db, self.query & query, 9937 ignore_common_filters=ignore_common_filters) 9938 else: 9939 return Set(self.db, query, 9940 ignore_common_filters=ignore_common_filters)
9941
9942 - def _count(self,distinct=None):
9943 return self.db._adapter._count(self.query,distinct)
9944
9945 - def _select(self, *fields, **attributes):
9946 adapter = self.db._adapter 9947 tablenames = adapter.tables(self.query, 9948 attributes.get('join',None), 9949 attributes.get('left',None), 9950 attributes.get('orderby',None), 9951 attributes.get('groupby',None)) 9952 fields = adapter.expand_all(fields, tablenames) 9953 return adapter._select(self.query,fields,attributes)
9954
9955 - def _delete(self):
9956 db = self.db 9957 tablename = db._adapter.get_table(self.query) 9958 return db._adapter._delete(tablename,self.query)
9959
9960 - def _update(self, **update_fields):
9961 db = self.db 9962 tablename = db._adapter.get_table(self.query) 9963 fields = db[tablename]._listify(update_fields,update=True) 9964 return db._adapter._update(tablename,self.query,fields)
9965
9966 - def as_dict(self, flat=False, sanitize=True):
9967 if flat: 9968 uid = dbname = uri = None 9969 codec = self.db._db_codec 9970 if not sanitize: 9971 uri, dbname, uid = (self.db._dbname, str(self.db), 9972 self.db._db_uid) 9973 d = {"query": self.query.as_dict(flat=flat)} 9974 d["db"] = {"uid": uid, "codec": codec, 9975 "name": dbname, "uri": uri} 9976 return d 9977 else: return self.__dict__
9978
9979 - def as_xml(self, sanitize=True):
9980 if have_serializers: 9981 xml = serializers.xml 9982 else: 9983 raise ImportError("No xml serializers available") 9984 d = self.as_dict(flat=True, sanitize=sanitize) 9985 return xml(d)
9986
9987 - def as_json(self, sanitize=True):
9988 if have_serializers: 9989 json = serializers.json 9990 else: 9991 raise ImportError("No json serializers available") 9992 d = self.as_dict(flat=True, sanitize=sanitize) 9993 return json(d)
9994
9995 - def parse(self, dquery):
9996 "Experimental: Turn a dictionary into a Query object" 9997 self.dquery = dquery 9998 return self.build(self.dquery)
9999
10000 - def build(self, d):
10001 "Experimental: see .parse()" 10002 op, first, second = (d["op"], d["first"], 10003 d.get("second", None)) 10004 left = right = built = None 10005 10006 if op in ("AND", "OR"): 10007 if not (type(first), type(second)) == (dict, dict): 10008 raise SyntaxError("Invalid AND/OR query") 10009 if op == "AND": 10010 built = self.build(first) & self.build(second) 10011 else: built = self.build(first) | self.build(second) 10012 10013 elif op == "NOT": 10014 if first is None: 10015 raise SyntaxError("Invalid NOT query") 10016 built = ~self.build(first) 10017 else: 10018 # normal operation (GT, EQ, LT, ...) 10019 for k, v in {"left": first, "right": second}.items(): 10020 if isinstance(v, dict) and v.get("op"): 10021 v = self.build(v) 10022 if isinstance(v, dict) and ("tablename" in v): 10023 v = self.db[v["tablename"]][v["fieldname"]] 10024 if k == "left": left = v 10025 else: right = v 10026 10027 if hasattr(self.db._adapter, op): 10028 opm = getattr(self.db._adapter, op) 10029 10030 if op == "EQ": built = left == right 10031 elif op == "NE": built = left != right 10032 elif op == "GT": built = left > right 10033 elif op == "GE": built = left >= right 10034 elif op == "LT": built = left < right 10035 elif op == "LE": built = left <= right 10036 elif op in ("JOIN", "LEFT_JOIN", "RANDOM", "ALLOW_NULL"): 10037 built = Expression(self.db, opm) 10038 elif op in ("LOWER", "UPPER", "EPOCH", "PRIMARY_KEY", 10039 "COALESCE_ZERO", "RAW", "INVERT"): 10040 built = Expression(self.db, opm, left) 10041 elif op in ("COUNT", "EXTRACT", "AGGREGATE", "SUBSTRING", 10042 "REGEXP", "LIKE", "ILIKE", "STARTSWITH", 10043 "ENDSWITH", "ADD", "SUB", "MUL", "DIV", 10044 "MOD", "AS", "ON", "COMMA", "NOT_NULL", 10045 "COALESCE", "CONTAINS", "BELONGS"): 10046 built = Expression(self.db, opm, left, right) 10047 # expression as string 10048 elif not (left or right): built = Expression(self.db, op) 10049 else: 10050 raise SyntaxError("Operator not supported: %s" % op) 10051 10052 return built
10053
10054 - def isempty(self):
10055 return not self.select(limitby=(0,1), orderby_on_limitby=False)
10056
10057 - def count(self,distinct=None, cache=None):
10058 db = self.db 10059 if cache: 10060 cache_model, time_expire = cache 10061 sql = self._count(distinct=distinct) 10062 key = db._uri + '/' + sql 10063 if len(key)>200: key = hashlib_md5(key).hexdigest() 10064 return cache_model( 10065 key, 10066 (lambda self=self,distinct=distinct: \ 10067 db._adapter.count(self.query,distinct)), 10068 time_expire) 10069 return db._adapter.count(self.query,distinct)
10070
10071 - def select(self, *fields, **attributes):
10072 adapter = self.db._adapter 10073 tablenames = adapter.tables(self.query, 10074 attributes.get('join',None), 10075 attributes.get('left',None), 10076 attributes.get('orderby',None), 10077 attributes.get('groupby',None)) 10078 fields = adapter.expand_all(fields, tablenames) 10079 return adapter.select(self.query,fields,attributes)
10080
10081 - def nested_select(self,*fields,**attributes):
10082 return Expression(self.db,self._select(*fields,**attributes))
10083
10084 - def delete(self):
10085 db = self.db 10086 tablename = db._adapter.get_table(self.query) 10087 table = db[tablename] 10088 if any(f(self) for f in table._before_delete): return 0 10089 ret = db._adapter.delete(tablename,self.query) 10090 ret and [f(self) for f in table._after_delete] 10091 return ret
10092
10093 - def update(self, **update_fields):
10094 db = self.db 10095 tablename = db._adapter.get_table(self.query) 10096 table = db[tablename] 10097 table._attempt_upload(update_fields) 10098 if any(f(self,update_fields) for f in table._before_update): 10099 return 0 10100 fields = table._listify(update_fields,update=True) 10101 if not fields: 10102 raise SyntaxError("No fields to update") 10103 ret = db._adapter.update("%s" % table,self.query,fields) 10104 ret and [f(self,update_fields) for f in table._after_update] 10105 return ret
10106
10107 - def update_naive(self, **update_fields):
10108 """ 10109 same as update but does not call table._before_update and _after_update 10110 """ 10111 tablename = self.db._adapter.get_table(self.query) 10112 table = self.db[tablename] 10113 fields = table._listify(update_fields,update=True) 10114 if not fields: raise SyntaxError("No fields to update") 10115 10116 ret = self.db._adapter.update("%s" % table,self.query,fields) 10117 return ret
10118
10119 - def validate_and_update(self, **update_fields):
10120 tablename = self.db._adapter.get_table(self.query) 10121 response = Row() 10122 response.errors = Row() 10123 new_fields = copy.copy(update_fields) 10124 for key,value in update_fields.iteritems(): 10125 value,error = self.db[tablename][key].validate(value) 10126 if error: 10127 response.errors[key] = error 10128 else: 10129 new_fields[key] = value 10130 table = self.db[tablename] 10131 if response.errors: 10132 response.updated = None 10133 else: 10134 if not any(f(self,new_fields) for f in table._before_update): 10135 fields = table._listify(new_fields,update=True) 10136 if not fields: raise SyntaxError("No fields to update") 10137 ret = self.db._adapter.update(tablename,self.query,fields) 10138 ret and [f(self,new_fields) for f in table._after_update] 10139 else: 10140 ret = 0 10141 response.updated = ret 10142 return response
10143
10144 - def delete_uploaded_files(self, upload_fields=None):
10145 table = self.db[self.db._adapter.tables(self.query)[0]] 10146 # ## mind uploadfield==True means file is not in DB 10147 if upload_fields: 10148 fields = upload_fields.keys() 10149 else: 10150 fields = table.fields 10151 fields = [f for f in fields if table[f].type == 'upload' 10152 and table[f].uploadfield == True 10153 and table[f].autodelete] 10154 if not fields: 10155 return False 10156 for record in self.select(*[table[f] for f in fields]): 10157 for fieldname in fields: 10158 field = table[fieldname] 10159 oldname = record.get(fieldname, None) 10160 if not oldname: 10161 continue 10162 if upload_fields and oldname == upload_fields[fieldname]: 10163 continue 10164 if field.custom_delete: 10165 field.custom_delete(oldname) 10166 else: 10167 uploadfolder = field.uploadfolder 10168 if not uploadfolder: 10169 uploadfolder = pjoin( 10170 self.db._adapter.folder, '..', 'uploads') 10171 if field.uploadseparate: 10172 items = oldname.split('.') 10173 uploadfolder = pjoin( 10174 uploadfolder, 10175 "%s.%s" % (items[0], items[1]), 10176 items[2][:2]) 10177 oldpath = pjoin(uploadfolder, oldname) 10178 if exists(oldpath): 10179 os.unlink(oldpath) 10180 return False
10181
10182 -class RecordUpdater(object):
10183 - def __init__(self, colset, table, id):
10184 self.colset, self.db, self.tablename, self.id = \ 10185 colset, table._db, table._tablename, id
10186
10187 - def __call__(self, **fields):
10188 colset, db, tablename, id = self.colset, self.db, self.tablename, self.id 10189 table = db[tablename] 10190 newfields = fields or dict(colset) 10191 for fieldname in newfields.keys(): 10192 if not fieldname in table.fields or table[fieldname].type=='id': 10193 del newfields[fieldname] 10194 table._db(table._id==id,ignore_common_filters=True).update(**newfields) 10195 colset.update(newfields) 10196 return colset
10197
10198 -class RecordDeleter(object):
10199 - def __init__(self, table, id):
10200 self.db, self.tablename, self.id = table._db, table._tablename, id
10201 - def __call__(self):
10202 return self.db(self.db[self.tablename]._id==self.id).delete()
10203
10204 -class LazyReferenceGetter(object):
10205 - def __init__(self, table, id):
10206 self.db, self.tablename, self.id = table._db, table._tablename, id
10207 - def __call__(self, other_tablename):
10208 if self.db._lazy_tables is False: 10209 raise AttributeError() 10210 table = self.db[self.tablename] 10211 other_table = self.db[other_tablename] 10212 for rfield in table._referenced_by: 10213 if rfield.table == other_table: 10214 return LazySet(rfield, self.id) 10215 10216 raise AttributeError()
10217
10218 -class LazySet(object):
10219 - def __init__(self, field, id):
10220 self.db, self.tablename, self.fieldname, self.id = \ 10221 field.db, field._tablename, field.name, id
10222 - def _getset(self):
10223 query = self.db[self.tablename][self.fieldname]==self.id 10224 return Set(self.db,query)
10225 - def __repr__(self):
10226 return repr(self._getset())
10227 - def __call__(self, query, ignore_common_filters=False):
10228 return self._getset()(query, ignore_common_filters)
10229 - def _count(self,distinct=None):
10230 return self._getset()._count(distinct)
10231 - def _select(self, *fields, **attributes):
10232 return self._getset()._select(*fields,**attributes)
10233 - def _delete(self):
10234 return self._getset()._delete()
10235 - def _update(self, **update_fields):
10236 return self._getset()._update(**update_fields)
10237 - def isempty(self):
10238 return self._getset().isempty()
10239 - def count(self,distinct=None, cache=None):
10240 return self._getset().count(distinct,cache)
10241 - def select(self, *fields, **attributes):
10242 return self._getset().select(*fields,**attributes)
10243 - def nested_select(self,*fields,**attributes):
10244 return self._getset().nested_select(*fields,**attributes)
10245 - def delete(self):
10246 return self._getset().delete()
10247 - def update(self, **update_fields):
10248 return self._getset().update(**update_fields)
10249 - def update_naive(self, **update_fields):
10250 return self._getset().update_naive(**update_fields)
10251 - def validate_and_update(self, **update_fields):
10252 return self._getset().validate_and_update(**update_fields)
10253 - def delete_uploaded_files(self, upload_fields=None):
10254 return self._getset().delete_uploaded_files(upload_fields)
10255
10256 -class VirtualCommand(object):
10257 - def __init__(self,method,row):
10258 self.method=method 10259 self.row=row
10260 - def __call__(self,*args,**kwargs):
10261 return self.method(self.row,*args,**kwargs)
10262
10263 -def lazy_virtualfield(f):
10264 f.__lazy__ = True 10265 return f
10266
10267 -class Rows(object):
10268 10269 """ 10270 A wrapper for the return value of a select. It basically represents a table. 10271 It has an iterator and each row is represented as a dictionary. 10272 """ 10273 10274 # ## TODO: this class still needs some work to care for ID/OID 10275
10276 - def __init__( 10277 self, 10278 db=None, 10279 records=[], 10280 colnames=[], 10281 compact=True, 10282 rawrows=None 10283 ):
10284 self.db = db 10285 self.records = records 10286 self.colnames = colnames 10287 self.compact = compact 10288 self.response = rawrows
10289
10290 - def __repr__(self):
10291 return '<Rows (%s)>' % len(self.records)
10292
10293 - def setvirtualfields(self,**keyed_virtualfields):
10294 """ 10295 db.define_table('x',Field('number','integer')) 10296 if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)] 10297 10298 from gluon.dal import lazy_virtualfield 10299 10300 class MyVirtualFields(object): 10301 # normal virtual field (backward compatible, discouraged) 10302 def normal_shift(self): return self.x.number+1 10303 # lazy virtual field (because of @staticmethod) 10304 @lazy_virtualfield 10305 def lazy_shift(instance,row,delta=4): return row.x.number+delta 10306 db.x.virtualfields.append(MyVirtualFields()) 10307 10308 for row in db(db.x).select(): 10309 print row.number, row.normal_shift, row.lazy_shift(delta=7) 10310 """ 10311 if not keyed_virtualfields: 10312 return self 10313 for row in self.records: 10314 for (tablename,virtualfields) in keyed_virtualfields.iteritems(): 10315 attributes = dir(virtualfields) 10316 if not tablename in row: 10317 box = row[tablename] = Row() 10318 else: 10319 box = row[tablename] 10320 updated = False 10321 for attribute in attributes: 10322 if attribute[0] != '_': 10323 method = getattr(virtualfields,attribute) 10324 if hasattr(method,'__lazy__'): 10325 box[attribute]=VirtualCommand(method,row) 10326 elif type(method)==types.MethodType: 10327 if not updated: 10328 virtualfields.__dict__.update(row) 10329 updated = True 10330 box[attribute]=method() 10331 return self
10332
10333 - def __and__(self,other):
10334 if self.colnames!=other.colnames: 10335 raise Exception('Cannot & incompatible Rows objects') 10336 records = self.records+other.records 10337 return Rows(self.db,records,self.colnames)
10338
10339 - def __or__(self,other):
10340 if self.colnames!=other.colnames: 10341 raise Exception('Cannot | incompatible Rows objects') 10342 records = self.records 10343 records += [record for record in other.records \ 10344 if not record in records] 10345 return Rows(self.db,records,self.colnames)
10346
10347 - def __nonzero__(self):
10348 if len(self.records): 10349 return 1 10350 return 0
10351
10352 - def __len__(self):
10353 return len(self.records)
10354
10355 - def __getslice__(self, a, b):
10356 return Rows(self.db,self.records[a:b],self.colnames,compact=self.compact)
10357
10358 - def __getitem__(self, i):
10359 row = self.records[i] 10360 keys = row.keys() 10361 if self.compact and len(keys) == 1 and keys[0] != '_extra': 10362 return row[row.keys()[0]] 10363 return row
10364
10365 - def __iter__(self):
10366 """ 10367 iterator over records 10368 """ 10369 10370 for i in xrange(len(self)): 10371 yield self[i]
10372
10373 - def __str__(self):
10374 """ 10375 serializes the table into a csv file 10376 """ 10377 10378 s = StringIO.StringIO() 10379 self.export_to_csv_file(s) 10380 return s.getvalue()
10381
10382 - def first(self):
10383 if not self.records: 10384 return None 10385 return self[0]
10386
10387 - def last(self):
10388 if not self.records: 10389 return None 10390 return self[-1]
10391
10392 - def find(self,f,limitby=None):
10393 """ 10394 returns a new Rows object, a subset of the original object, 10395 filtered by the function f 10396 """ 10397 if not self: 10398 return Rows(self.db, [], self.colnames) 10399 records = [] 10400 if limitby: 10401 a,b = limitby 10402 else: 10403 a,b = 0,len(self) 10404 k = 0 10405 for row in self: 10406 if f(row): 10407 if a<=k: records.append(row) 10408 k += 1 10409 if k==b: break 10410 return Rows(self.db, records, self.colnames)
10411
10412 - def exclude(self, f):
10413 """ 10414 removes elements from the calling Rows object, filtered by the function f, 10415 and returns a new Rows object containing the removed elements 10416 """ 10417 if not self.records: 10418 return Rows(self.db, [], self.colnames) 10419 removed = [] 10420 i=0 10421 while i<len(self): 10422 row = self[i] 10423 if f(row): 10424 removed.append(self.records[i]) 10425 del self.records[i] 10426 else: 10427 i += 1 10428 return Rows(self.db, removed, self.colnames)
10429
10430 - def sort(self, f, reverse=False):
10431 """ 10432 returns a list of sorted elements (not sorted in place) 10433 """ 10434 rows = Rows(self.db,[],self.colnames,compact=False) 10435 rows.records = sorted(self,key=f,reverse=reverse) 10436 return rows
10437
10438 - def group_by_value(self, *fields, **args):
10439 """ 10440 regroups the rows, by one of the fields 10441 """ 10442 one_result = False 10443 if 'one_result' in args: 10444 one_result = args['one_result'] 10445 10446 def build_fields_struct(row, fields, num, groups): 10447 ''' helper function: 10448 ''' 10449 if num > len(fields)-1: 10450 if one_result: 10451 return row 10452 else: 10453 return [row] 10454 10455 key = fields[num] 10456 value = row[key] 10457 10458 if value not in groups: 10459 groups[value] = build_fields_struct(row, fields, num+1, {}) 10460 else: 10461 struct = build_fields_struct(row, fields, num+1, groups[ value ]) 10462 10463 # still have more grouping to do 10464 if type(struct) == type(dict()): 10465 groups[value].update() 10466 # no more grouping, first only is off 10467 elif type(struct) == type(list()): 10468 groups[value] += struct 10469 # no more grouping, first only on 10470 else: 10471 groups[value] = struct 10472 10473 return groups
10474 10475 if len(fields) == 0: 10476 return self 10477 10478 # if select returned no results 10479 if not self.records: 10480 return {} 10481 10482 grouped_row_group = dict() 10483 10484 # build the struct 10485 for row in self: 10486 build_fields_struct(row, fields, 0, grouped_row_group) 10487 10488 return grouped_row_group
10489
10490 - def render(self, i=None, fields=None):
10491 """ 10492 Takes an index and returns a copy of the indexed row with values 10493 transformed via the "represent" attributes of the associated fields. 10494 10495 If no index is specified, a generator is returned for iteration 10496 over all the rows. 10497 10498 fields -- a list of fields to transform (if None, all fields with 10499 "represent" attributes will be transformed). 10500 """ 10501 10502 10503 if i is None: 10504 return (self.repr(i, fields=fields) for i in range(len(self))) 10505 import sqlhtml 10506 row = copy.deepcopy(self.records[i]) 10507 keys = row.keys() 10508 tables = [f.tablename for f in fields] if fields \ 10509 else [k for k in keys if k != '_extra'] 10510 for table in tables: 10511 repr_fields = [f.name for f in fields if f.tablename == table] \ 10512 if fields else [k for k in row[table].keys() 10513 if (hasattr(self.db[table], k) and 10514 isinstance(self.db[table][k], Field) 10515 and self.db[table][k].represent)] 10516 for field in repr_fields: 10517 row[table][field] = sqlhtml.represent( 10518 self.db[table][field], row[table][field], row[table]) 10519 if self.compact and len(keys) == 1 and keys[0] != '_extra': 10520 return row[keys[0]] 10521 return row
10522
10523 - def as_list(self, 10524 compact=True, 10525 storage_to_dict=True, 10526 datetime_to_str=False, 10527 custom_types=None):
10528 """ 10529 returns the data as a list or dictionary. 10530 :param storage_to_dict: when True returns a dict, otherwise a list(default True) 10531 :param datetime_to_str: convert datetime fields as strings (default False) 10532 """ 10533 (oc, self.compact) = (self.compact, compact) 10534 if storage_to_dict: 10535 items = [item.as_dict(datetime_to_str, custom_types) for item in self] 10536 else: 10537 items = [item for item in self] 10538 self.compact = compact 10539 return items
10540 10541
10542 - def as_dict(self, 10543 key='id', 10544 compact=True, 10545 storage_to_dict=True, 10546 datetime_to_str=False, 10547 custom_types=None):
10548 """ 10549 returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False) 10550 10551 :param key: the name of the field to be used as dict key, normally the id 10552 :param compact: ? (default True) 10553 :param storage_to_dict: when True returns a dict, otherwise a list(default True) 10554 :param datetime_to_str: convert datetime fields as strings (default False) 10555 """ 10556 10557 # test for multiple rows 10558 multi = False 10559 f = self.first() 10560 if f and isinstance(key, basestring): 10561 multi = any([isinstance(v, f.__class__) for v in f.values()]) 10562 if (not "." in key) and multi: 10563 # No key provided, default to int indices 10564 def new_key(): 10565 i = 0 10566 while True: 10567 yield i 10568 i += 1
10569 key_generator = new_key() 10570 key = lambda r: key_generator.next() 10571 10572 rows = self.as_list(compact, storage_to_dict, datetime_to_str, custom_types) 10573 if isinstance(key,str) and key.count('.')==1: 10574 (table, field) = key.split('.') 10575 return dict([(r[table][field],r) for r in rows]) 10576 elif isinstance(key,str): 10577 return dict([(r[key],r) for r in rows]) 10578 else: 10579 return dict([(key(r),r) for r in rows]) 10580
10581 - def export_to_csv_file(self, ofile, null='<NULL>', *args, **kwargs):
10582 """ 10583 export data to csv, the first line contains the column names 10584 10585 :param ofile: where the csv must be exported to 10586 :param null: how null values must be represented (default '<NULL>') 10587 :param delimiter: delimiter to separate values (default ',') 10588 :param quotechar: character to use to quote string values (default '"') 10589 :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL) 10590 :param represent: use the fields .represent value (default False) 10591 :param colnames: list of column names to use (default self.colnames) 10592 This will only work when exporting rows objects!!!! 10593 DO NOT use this with db.export_to_csv() 10594 """ 10595 delimiter = kwargs.get('delimiter', ',') 10596 quotechar = kwargs.get('quotechar', '"') 10597 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL) 10598 represent = kwargs.get('represent', False) 10599 writer = csv.writer(ofile, delimiter=delimiter, 10600 quotechar=quotechar, quoting=quoting) 10601 colnames = kwargs.get('colnames', self.colnames) 10602 write_colnames = kwargs.get('write_colnames',True) 10603 # a proper csv starting with the column names 10604 if write_colnames: 10605 writer.writerow(colnames) 10606 10607 def none_exception(value): 10608 """ 10609 returns a cleaned up value that can be used for csv export: 10610 - unicode text is encoded as such 10611 - None values are replaced with the given representation (default <NULL>) 10612 """ 10613 if value is None: 10614 return null 10615 elif isinstance(value, unicode): 10616 return value.encode('utf8') 10617 elif isinstance(value,Reference): 10618 return long(value) 10619 elif hasattr(value, 'isoformat'): 10620 return value.isoformat()[:19].replace('T', ' ') 10621 elif isinstance(value, (list,tuple)): # for type='list:..' 10622 return bar_encode(value) 10623 return value
10624 10625 for record in self: 10626 row = [] 10627 for col in colnames: 10628 if not REGEX_TABLE_DOT_FIELD.match(col): 10629 row.append(record._extra[col]) 10630 else: 10631 (t, f) = col.split('.') 10632 field = self.db[t][f] 10633 if isinstance(record.get(t, None), (Row,dict)): 10634 value = record[t][f] 10635 else: 10636 value = record[f] 10637 if field.type=='blob' and not value is None: 10638 value = base64.b64encode(value) 10639 elif represent and field.represent: 10640 value = field.represent(value) 10641 row.append(none_exception(value)) 10642 writer.writerow(row) 10643
10644 - def xml(self,strict=False,row_name='row',rows_name='rows'):
10645 """ 10646 serializes the table using sqlhtml.SQLTABLE (if present) 10647 """ 10648 10649 if strict: 10650 ncols = len(self.colnames) 10651 return '<%s>\n%s\n</%s>' % (rows_name, 10652 '\n'.join(row.as_xml(row_name=row_name, 10653 colnames=self.colnames) for 10654 row in self), rows_name) 10655 10656 import sqlhtml 10657 return sqlhtml.SQLTABLE(self).xml()
10658
10659 - def as_xml(self,row_name='row',rows_name='rows'):
10660 return self.xml(strict=True, row_name=row_name, rows_name=rows_name)
10661
10662 - def as_json(self, mode='object', default=None):
10663 """ 10664 serializes the rows to a JSON list or object with objects 10665 mode='object' is not implemented (should return a nested 10666 object structure) 10667 """ 10668 10669 items = [record.as_json(mode=mode, default=default, 10670 serialize=False, 10671 colnames=self.colnames) for 10672 record in self] 10673 10674 if have_serializers: 10675 return serializers.json(items, 10676 default=default or 10677 serializers.custom_json) 10678 elif simplejson: 10679 return simplejson.dumps(items) 10680 else: 10681 raise RuntimeError("missing simplejson")
10682 10683 # for consistent naming yet backwards compatible 10684 as_csv = __str__ 10685 json = as_json 10686
10687 10688 ################################################################################ 10689 # dummy function used to define some doctests 10690 ################################################################################ 10691 10692 -def test_all():
10693 """ 10694 10695 >>> if len(sys.argv)<2: db = DAL("sqlite://test.db") 10696 >>> if len(sys.argv)>1: db = DAL(sys.argv[1]) 10697 >>> tmp = db.define_table('users',\ 10698 Field('stringf', 'string', length=32, required=True),\ 10699 Field('booleanf', 'boolean', default=False),\ 10700 Field('passwordf', 'password', notnull=True),\ 10701 Field('uploadf', 'upload'),\ 10702 Field('blobf', 'blob'),\ 10703 Field('integerf', 'integer', unique=True),\ 10704 Field('doublef', 'double', unique=True,notnull=True),\ 10705 Field('jsonf', 'json'),\ 10706 Field('datef', 'date', default=datetime.date.today()),\ 10707 Field('timef', 'time'),\ 10708 Field('datetimef', 'datetime'),\ 10709 migrate='test_user.table') 10710 10711 Insert a field 10712 10713 >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\ 10714 uploadf=None, integerf=5, doublef=3.14,\ 10715 jsonf={"j": True},\ 10716 datef=datetime.date(2001, 1, 1),\ 10717 timef=datetime.time(12, 30, 15),\ 10718 datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15)) 10719 1 10720 10721 Drop the table 10722 10723 >>> db.users.drop() 10724 10725 Examples of insert, select, update, delete 10726 10727 >>> tmp = db.define_table('person',\ 10728 Field('name'),\ 10729 Field('birth','date'),\ 10730 migrate='test_person.table') 10731 >>> person_id = db.person.insert(name='Marco',birth='2005-06-22') 10732 >>> person_id = db.person.insert(name='Massimo',birth='1971-12-21') 10733 10734 commented len(db().select(db.person.ALL)) 10735 commented 2 10736 10737 >>> me = db(db.person.id==person_id).select()[0] # test select 10738 >>> me.name 10739 'Massimo' 10740 >>> db.person[2].name 10741 'Massimo' 10742 >>> db.person(2).name 10743 'Massimo' 10744 >>> db.person(name='Massimo').name 10745 'Massimo' 10746 >>> db.person(db.person.name=='Massimo').name 10747 'Massimo' 10748 >>> row = db.person[2] 10749 >>> row.name == row['name'] == row['person.name'] == row('person.name') 10750 True 10751 >>> db(db.person.name=='Massimo').update(name='massimo') # test update 10752 1 10753 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete 10754 1 10755 10756 Update a single record 10757 10758 >>> me.update_record(name="Max") 10759 <Row {'name': 'Max', 'birth': datetime.date(1971, 12, 21), 'id': 2}> 10760 >>> me.name 10761 'Max' 10762 10763 Examples of complex search conditions 10764 10765 >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select()) 10766 1 10767 >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select()) 10768 1 10769 >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select()) 10770 1 10771 >>> me = db(db.person.id==person_id).select(db.person.name)[0] 10772 >>> me.name 10773 'Max' 10774 10775 Examples of search conditions using extract from date/datetime/time 10776 10777 >>> len(db(db.person.birth.month()==12).select()) 10778 1 10779 >>> len(db(db.person.birth.year()>1900).select()) 10780 1 10781 10782 Example of usage of NULL 10783 10784 >>> len(db(db.person.birth==None).select()) ### test NULL 10785 0 10786 >>> len(db(db.person.birth!=None).select()) ### test NULL 10787 1 10788 10789 Examples of search conditions using lower, upper, and like 10790 10791 >>> len(db(db.person.name.upper()=='MAX').select()) 10792 1 10793 >>> len(db(db.person.name.like('%ax')).select()) 10794 1 10795 >>> len(db(db.person.name.upper().like('%AX')).select()) 10796 1 10797 >>> len(db(~db.person.name.upper().like('%AX')).select()) 10798 0 10799 10800 orderby, groupby and limitby 10801 10802 >>> people = db().select(db.person.name, orderby=db.person.name) 10803 >>> order = db.person.name|~db.person.birth 10804 >>> people = db().select(db.person.name, orderby=order) 10805 10806 >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name) 10807 10808 >>> people = db().select(db.person.name, orderby=order, limitby=(0,100)) 10809 10810 Example of one 2 many relation 10811 10812 >>> tmp = db.define_table('dog',\ 10813 Field('name'),\ 10814 Field('birth','date'),\ 10815 Field('owner',db.person),\ 10816 migrate='test_dog.table') 10817 >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id) 10818 1 10819 10820 A simple JOIN 10821 10822 >>> len(db(db.dog.owner==db.person.id).select()) 10823 1 10824 10825 >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id))) 10826 1 10827 10828 Drop tables 10829 10830 >>> db.dog.drop() 10831 >>> db.person.drop() 10832 10833 Example of many 2 many relation and Set 10834 10835 >>> tmp = db.define_table('author', Field('name'),\ 10836 migrate='test_author.table') 10837 >>> tmp = db.define_table('paper', Field('title'),\ 10838 migrate='test_paper.table') 10839 >>> tmp = db.define_table('authorship',\ 10840 Field('author_id', db.author),\ 10841 Field('paper_id', db.paper),\ 10842 migrate='test_authorship.table') 10843 >>> aid = db.author.insert(name='Massimo') 10844 >>> pid = db.paper.insert(title='QCD') 10845 >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid) 10846 10847 Define a Set 10848 10849 >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id)) 10850 >>> rows = authored_papers.select(db.author.name, db.paper.title) 10851 >>> for row in rows: print row.author.name, row.paper.title 10852 Massimo QCD 10853 10854 Example of search condition using belongs 10855 10856 >>> set = (1, 2, 3) 10857 >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL) 10858 >>> print rows[0].title 10859 QCD 10860 10861 Example of search condition using nested select 10862 10863 >>> nested_select = db()._select(db.authorship.paper_id) 10864 >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL) 10865 >>> print rows[0].title 10866 QCD 10867 10868 Example of expressions 10869 10870 >>> mynumber = db.define_table('mynumber', Field('x', 'integer')) 10871 >>> db(mynumber).delete() 10872 0 10873 >>> for i in range(10): tmp = mynumber.insert(x=i) 10874 >>> db(mynumber).select(mynumber.x.sum())[0](mynumber.x.sum()) 10875 45 10876 10877 >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2) 10878 5 10879 10880 Output in csv 10881 10882 >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip() 10883 author.name,paper.title\r 10884 Massimo,QCD 10885 10886 Delete all leftover tables 10887 10888 >>> DAL.distributed_transaction_commit(db) 10889 10890 >>> db.mynumber.drop() 10891 >>> db.authorship.drop() 10892 >>> db.author.drop() 10893 >>> db.paper.drop() 10894 """
10895 ################################################################################ 10896 # deprecated since the new DAL; here only for backward compatibility 10897 ################################################################################ 10898 10899 SQLField = Field 10900 SQLTable = Table 10901 SQLXorable = Expression 10902 SQLQuery = Query 10903 SQLSet = Set 10904 SQLRows = Rows 10905 SQLStorage = Row 10906 SQLDB = DAL 10907 GQLDB = DAL 10908 DAL.Field = Field # was necessary in gluon/globals.py session.connect 10909 DAL.Table = Table # was necessary in gluon/globals.py session.connect
10910 10911 ################################################################################ 10912 # Geodal utils 10913 ################################################################################ 10914 10915 -def geoPoint(x,y):
10916 return "POINT (%f %f)" % (x,y)
10917
10918 -def geoLine(*line):
10919 return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
10920
10921 -def geoPolygon(*line):
10922 return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)
10923 10924 ################################################################################ 10925 # run tests 10926 ################################################################################ 10927 10928 if __name__ == '__main__': 10929 import doctest 10930 doctest.testmod() 10931